From 65cbc7cb2512cd96cfa3461a816eec6f2da36b08 Mon Sep 17 00:00:00 2001 From: John Howard Date: Mon, 1 Jun 2015 16:42:27 -0700 Subject: [PATCH] Windows: Docker build starting to work Signed-off-by: John Howard --- archive/archive.go | 22 +++-- archive/changes.go | 22 +++-- archive/changes_other.go | 17 +++- archive/diff.go | 32 ++++++- chrootarchive/archive.go | 111 +----------------------- chrootarchive/archive_unix.go | 71 +++++++++++++++ chrootarchive/archive_windows.go | 15 ++++ chrootarchive/{diff.go => diff_unix.go} | 36 ++++---- chrootarchive/diff_windows.go | 32 +++++++ chrootarchive/{init.go => init_unix.go} | 4 +- chrootarchive/init_windows.go | 4 + 11 files changed, 219 insertions(+), 147 deletions(-) rename chrootarchive/{diff.go => diff_unix.go} (75%) create mode 100644 chrootarchive/diff_windows.go rename chrootarchive/{init.go => init_unix.go} (95%) create mode 100644 chrootarchive/init_windows.go diff --git a/archive/archive.go b/archive/archive.go index cde4de5..2722c99 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -452,6 +452,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } seen[relFilePath] = true + // TODO Windows: Verify if this needs to be os.Pathseparator // Rename the base resource if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) { renamedRelFilePath = relFilePath @@ -503,7 +504,8 @@ loop: } // Normalize name, for safety and for a simple is-root check - // This keeps "../" as-is, but normalizes "/../" to "/" + // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: + // This keeps "..\" as-is, but normalizes "\..\" to "\". hdr.Name = filepath.Clean(hdr.Name) for _, exclude := range options.ExcludePatterns { @@ -512,7 +514,10 @@ loop: } } - if !strings.HasSuffix(hdr.Name, "/") { + // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in + // the filepath format for the OS on which the daemon is running. Hence + // the check for a slash-suffix MUST be done in an OS-agnostic way. + if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { // Not the root directory, ensure that the parent directory exists parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) @@ -529,7 +534,7 @@ loop: if err != nil { return err } - if strings.HasPrefix(rel, "../") { + if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } @@ -658,10 +663,13 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { if err != nil { return err } + if srcSt.IsDir() { return fmt.Errorf("Can't copy a directory") } - // Clean up the trailing slash + + // Clean up the trailing slash. This must be done in an operating + // system specific manner. if dst[len(dst)-1] == os.PathSeparator { dst = filepath.Join(dst, filepath.Base(src)) } @@ -709,8 +717,10 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { // for a single file. It copies a regular file from path `src` to // path `dst`, and preserves all its metadata. // -// If `dst` ends with a trailing slash '/', the final destination path -// will be `dst/base(src)`. +// Destination handling is in an operating specific manner depending +// where the daemon is running. If `dst` ends with a trailing slash +// the final destination path will be `dst/base(src)` (Linux) or +// `dst\base(src)` (Windows). func CopyFileWithTar(src, dst string) (err error) { return defaultArchiver.CopyFileWithTar(src, dst) } diff --git a/archive/changes.go b/archive/changes.go index 15bd3dd..689d9a2 100644 --- a/archive/changes.go +++ b/archive/changes.go @@ -84,15 +84,17 @@ func Changes(layers []string, rw string) ([]Change, error) { if err != nil { return err } - path = filepath.Join("/", path) + + // As this runs on the daemon side, file paths are OS specific. + path = filepath.Join(string(os.PathSeparator), path) // Skip root - if path == "/" { + if path == string(os.PathSeparator) { return nil } // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched { return err } @@ -169,12 +171,13 @@ type FileInfo struct { } func (root *FileInfo) LookUp(path string) *FileInfo { + // As this runs on the daemon side, file paths are OS specific. parent := root - if path == "/" { + if path == string(os.PathSeparator) { return root } - pathElements := strings.Split(path, "/") + pathElements := strings.Split(path, string(os.PathSeparator)) for _, elem := range pathElements { if elem != "" { child := parent.children[elem] @@ -189,7 +192,8 @@ func (root *FileInfo) LookUp(path string) *FileInfo { func (info *FileInfo) path() string { if info.parent == nil { - return "/" + // As this runs on the daemon side, file paths are OS specific. + return string(os.PathSeparator) } return filepath.Join(info.parent.path(), info.name) } @@ -257,7 +261,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { // If there were changes inside this directory, we need to add it, even if the directory // itself wasn't changed. This is needed to properly save and restore filesystem permissions. - if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" { + // As this runs on the daemon side, file paths are OS specific. + if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) { change := Change{ Path: info.path(), Kind: ChangeModify, @@ -279,8 +284,9 @@ func (info *FileInfo) Changes(oldInfo *FileInfo) []Change { } func newRootFileInfo() *FileInfo { + // As this runs on the daemon side, file paths are OS specific. root := &FileInfo{ - name: "/", + name: string(os.PathSeparator), children: make(map[string]*FileInfo), } return root diff --git a/archive/changes_other.go b/archive/changes_other.go index e2d9c23..da70ed3 100644 --- a/archive/changes_other.go +++ b/archive/changes_other.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path/filepath" + "runtime" + "strings" "github.com/docker/docker/pkg/system" ) @@ -48,9 +50,20 @@ func collectFileInfo(sourceDir string) (*FileInfo, error) { if err != nil { return err } - relPath = filepath.Join("/", relPath) - if relPath == "/" { + // As this runs on the daemon side, file paths are OS specific. + relPath = filepath.Join(string(os.PathSeparator), relPath) + + // See https://github.com/golang/go/issues/9168 - bug in filepath.Join. + // Temporary workaround. If the returned path starts with two backslashes, + // trim it down to a single backslash. Only relevant on Windows. + if runtime.GOOS == "windows" { + if strings.HasPrefix(relPath, `\\`) { + relPath = relPath[1:] + } + } + + if relPath == string(os.PathSeparator) { return nil } diff --git a/archive/diff.go b/archive/diff.go index fd49460..af8d3ee 100644 --- a/archive/diff.go +++ b/archive/diff.go @@ -7,9 +7,11 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "syscall" + "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/system" ) @@ -40,12 +42,35 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) { // Normalize name, for safety and for a simple is-root check hdr.Name = filepath.Clean(hdr.Name) - if !strings.HasSuffix(hdr.Name, "/") { + // Windows does not support filenames with colons in them. Ignore + // these files. This is not a problem though (although it might + // appear that it is). Let's suppose a client is running docker pull. + // The daemon it points to is Windows. Would it make sense for the + // client to be doing a docker pull Ubuntu for example (which has files + // with colons in the name under /usr/share/man/man3)? No, absolutely + // not as it would really only make sense that they were pulling a + // Windows image. However, for development, it is necessary to be able + // to pull Linux images which are in the repository. + // + // TODO Windows. Once the registry is aware of what images are Windows- + // specific or Linux-specific, this warning should be changed to an error + // to cater for the situation where someone does manage to upload a Linux + // image but have it tagged as Windows inadvertantly. + if runtime.GOOS == "windows" { + if strings.Contains(hdr.Name, ":") { + logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name) + continue + } + } + + // Note as these operations are platform specific, so must the slash be. + if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { // Not the root directory, ensure that the parent directory exists. // This happened in some tests where an image had a tarfile without any // parent directories. parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) + if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = system.MkdirAll(parentPath, 0600) if err != nil { @@ -74,13 +99,14 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) { } continue } - path := filepath.Join(dest, hdr.Name) rel, err := filepath.Rel(dest, path) if err != nil { return 0, err } - if strings.HasPrefix(rel, "../") { + + // Note as these operations are platform specific, so must the slash be. + if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } base := filepath.Base(path) diff --git a/chrootarchive/archive.go b/chrootarchive/archive.go index 06db8b2..dffbec1 100644 --- a/chrootarchive/archive.go +++ b/chrootarchive/archive.go @@ -1,68 +1,23 @@ package chrootarchive import ( - "bytes" - "encoding/json" - "flag" "fmt" "io" "os" "path/filepath" - "runtime" "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/system" ) var chrootArchiver = &archive.Archiver{Untar: Untar} -func untar() { - runtime.LockOSThread() - flag.Parse() - - var options *archive.TarOptions - - if runtime.GOOS != "windows" { - //read the options from the pipe "ExtraFiles" - if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { - fatal(err) - } - } else { - if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil { - fatal(err) - } - } - - if err := chroot(flag.Arg(0)); err != nil { - fatal(err) - } - - // Explanation of Windows difference. Windows does not support chroot. - // untar() is a helper function for the command line in the format - // "docker docker-untar directory input". In Windows, directory will be - // something like \docker-buildnnnnnnnnn. So, just use that directory - // directly instead. - // - // One example of where this is used is in the docker build command where the - // dockerfile will be unpacked to the machine on which the daemon runs. - rootPath := "/" - if runtime.GOOS == "windows" { - rootPath = flag.Arg(0) - } - if err := archive.Unpack(os.Stdin, rootPath, options); err != nil { - fatal(err) - } - // fully consume stdin in case it is zero padded - flush(os.Stdin) - os.Exit(0) -} - // Untar reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { + if tarArchive == nil { return fmt.Errorf("Empty archive") } @@ -84,67 +39,9 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error if err != nil { return err } - - var data []byte - var r, w *os.File defer decompressedArchive.Close() - if runtime.GOOS != "windows" { - // We can't pass a potentially large exclude list directly via cmd line - // because we easily overrun the kernel's max argument/environment size - // when the full image list is passed (e.g. when this is used by - // `docker load`). We will marshall the options via a pipe to the - // child - - // This solution won't work on Windows as it will fail in golang - // exec_windows.go as at the lowest layer because attr.Files > 3 - r, w, err = os.Pipe() - if err != nil { - return fmt.Errorf("Untar pipe failure: %v", err) - } - } else { - // We can't pass the exclude list directly via cmd line - // because we easily overrun the shell max argument list length - // when the full image list is passed (e.g. when this is used - // by `docker load`). Instead we will add the JSON marshalled - // and placed in the env, which has significantly larger - // max size - data, err = json.Marshal(options) - if err != nil { - return fmt.Errorf("Untar json encode: %v", err) - } - } - - cmd := reexec.Command("docker-untar", dest) - cmd.Stdin = decompressedArchive - - if runtime.GOOS != "windows" { - cmd.ExtraFiles = append(cmd.ExtraFiles, r) - output := bytes.NewBuffer(nil) - cmd.Stdout = output - cmd.Stderr = output - - if err := cmd.Start(); err != nil { - return fmt.Errorf("Untar error on re-exec cmd: %v", err) - } - //write the options to the pipe for the untar exec to read - if err := json.NewEncoder(w).Encode(options); err != nil { - return fmt.Errorf("Untar json encode to pipe failed: %v", err) - } - w.Close() - - if err := cmd.Wait(); err != nil { - return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output) - } - return nil - } - cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data)) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("Untar %s %s", err, out) - } - return nil - + return invokeUnpack(decompressedArchive, dest, options) } // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. @@ -165,8 +62,8 @@ func CopyWithTar(src, dst string) error { // for a single file. It copies a regular file from path `src` to // path `dst`, and preserves all its metadata. // -// If `dst` ends with a trailing slash '/', the final destination path -// will be `dst/base(src)`. +// If `dst` ends with a trailing slash '/' ('\' on Windows), the final +// destination path will be `dst/base(src)` or `dst\base(src)` func CopyFileWithTar(src, dst string) (err error) { return chrootArchiver.CopyFileWithTar(src, dst) } diff --git a/chrootarchive/archive_unix.go b/chrootarchive/archive_unix.go index 62c46f0..d60718d 100644 --- a/chrootarchive/archive_unix.go +++ b/chrootarchive/archive_unix.go @@ -3,7 +3,17 @@ package chrootarchive import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "runtime" "syscall" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/reexec" ) func chroot(path string) error { @@ -12,3 +22,64 @@ func chroot(path string) error { } return syscall.Chdir("/") } + +// untar is the entry-point for docker-untar on re-exec. This is not used on +// Windows as it does not support chroot, hence no point sandboxing through +// chroot and rexec. +func untar() { + runtime.LockOSThread() + flag.Parse() + + var options *archive.TarOptions + + //read the options from the pipe "ExtraFiles" + if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { + fatal(err) + } + + if err := chroot(flag.Arg(0)); err != nil { + fatal(err) + } + + if err := archive.Unpack(os.Stdin, "/", options); err != nil { + fatal(err) + } + // fully consume stdin in case it is zero padded + flush(os.Stdin) + os.Exit(0) +} + +func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error { + + // We can't pass a potentially large exclude list directly via cmd line + // because we easily overrun the kernel's max argument/environment size + // when the full image list is passed (e.g. when this is used by + // `docker load`). We will marshall the options via a pipe to the + // child + r, w, err := os.Pipe() + if err != nil { + return fmt.Errorf("Untar pipe failure: %v", err) + } + + cmd := reexec.Command("docker-untar", dest) + cmd.Stdin = decompressedArchive + + cmd.ExtraFiles = append(cmd.ExtraFiles, r) + output := bytes.NewBuffer(nil) + cmd.Stdout = output + cmd.Stderr = output + + if err := cmd.Start(); err != nil { + return fmt.Errorf("Untar error on re-exec cmd: %v", err) + } + //write the options to the pipe for the untar exec to read + if err := json.NewEncoder(w).Encode(options); err != nil { + return fmt.Errorf("Untar json encode to pipe failed: %v", err) + } + w.Close() + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output) + } + return nil +} diff --git a/chrootarchive/archive_windows.go b/chrootarchive/archive_windows.go index dccfb87..c44556c 100644 --- a/chrootarchive/archive_windows.go +++ b/chrootarchive/archive_windows.go @@ -1,6 +1,21 @@ package chrootarchive +import ( + "io" + + "github.com/docker/docker/pkg/archive" +) + // chroot is not supported by Windows func chroot(path string) error { return nil } + +func invokeUnpack(decompressedArchive io.ReadCloser, + dest string, + options *archive.TarOptions) error { + // Windows is different to Linux here because Windows does not support + // chroot. Hence there is no point sandboxing a chrooted process to + // do the unpack. We call inline instead within the daemon process. + return archive.Unpack(decompressedArchive, dest, options) +} diff --git a/chrootarchive/diff.go b/chrootarchive/diff_unix.go similarity index 75% rename from chrootarchive/diff.go rename to chrootarchive/diff_unix.go index c99aed0..f8678ab 100644 --- a/chrootarchive/diff.go +++ b/chrootarchive/diff_unix.go @@ -1,3 +1,5 @@ +//+build !windows + package chrootarchive import ( @@ -19,41 +21,35 @@ type applyLayerResponse struct { LayerSize int64 `json:"layerSize"` } +// applyLayer is the entry-point for docker-applylayer on re-exec. This is not +// used on Windows as it does not support chroot, hence no point sandboxing +// through chroot and rexec. func applyLayer() { var ( - root = "/" tmpDir = "" err error ) - runtime.LockOSThread() flag.Parse() - if runtime.GOOS != "windows" { - if err := chroot(flag.Arg(0)); err != nil { - fatal(err) - } - - // We need to be able to set any perms - oldmask, err := system.Umask(0) - defer system.Umask(oldmask) - if err != nil { - fatal(err) - } - } else { - // As Windows does not support chroot or umask, we use the directory - // passed in which will be \docker-buildnnnnnnnn instead of - // the 'chroot-root', "/" - root = flag.Arg(0) + if err := chroot(flag.Arg(0)); err != nil { + fatal(err) } - if tmpDir, err = ioutil.TempDir(root, "temp-docker-extract"); err != nil { + // We need to be able to set any perms + oldmask, err := system.Umask(0) + defer system.Umask(oldmask) + if err != nil { + fatal(err) + } + + if tmpDir, err = ioutil.TempDir("/", "temp-docker-extract"); err != nil { fatal(err) } os.Setenv("TMPDIR", tmpDir) - size, err := archive.UnpackLayer(root, os.Stdin) + size, err := archive.UnpackLayer("/", os.Stdin) os.RemoveAll(tmpDir) if err != nil { fatal(err) diff --git a/chrootarchive/diff_windows.go b/chrootarchive/diff_windows.go new file mode 100644 index 0000000..72289c8 --- /dev/null +++ b/chrootarchive/diff_windows.go @@ -0,0 +1,32 @@ +package chrootarchive + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/docker/docker/pkg/archive" +) + +func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) { + dest = filepath.Clean(dest) + decompressed, err := archive.DecompressStream(layer) + if err != nil { + return 0, err + } + defer decompressed.Close() + + tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract") + if err != nil { + return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err) + } + + s, err := archive.UnpackLayer(dest, decompressed) + os.RemoveAll(tmpDir) + if err != nil { + return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest) + } + + return s, nil +} diff --git a/chrootarchive/init.go b/chrootarchive/init_unix.go similarity index 95% rename from chrootarchive/init.go rename to chrootarchive/init_unix.go index 4116026..49fcacc 100644 --- a/chrootarchive/init.go +++ b/chrootarchive/init_unix.go @@ -1,3 +1,5 @@ +// +build !windows + package chrootarchive import ( @@ -10,8 +12,8 @@ import ( ) func init() { - reexec.Register("docker-untar", untar) reexec.Register("docker-applyLayer", applyLayer) + reexec.Register("docker-untar", untar) } func fatal(err error) { diff --git a/chrootarchive/init_windows.go b/chrootarchive/init_windows.go new file mode 100644 index 0000000..fa17c9b --- /dev/null +++ b/chrootarchive/init_windows.go @@ -0,0 +1,4 @@ +package chrootarchive + +func init() { +}