Windows: Docker build starting to work

Signed-off-by: John Howard <jhoward@microsoft.com>

archive/archive.go

@@ -452,6 +452,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 				}
 				seen[relFilePath] = true
 
+				// TODO Windows: Verify if this needs to be os.Pathseparator
 				// Rename the base resource
 				if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) {
 					renamedRelFilePath = relFilePath
@@ -503,7 +504,8 @@ loop:
 		}
 
 		// Normalize name, for safety and for a simple is-root check
-		// This keeps "../" as-is, but normalizes "/../" to "/"
+		// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
+		// This keeps "..\" as-is, but normalizes "\..\" to "\".
 		hdr.Name = filepath.Clean(hdr.Name)
 
 		for _, exclude := range options.ExcludePatterns {
@@ -512,7 +514,10 @@ loop:
 			}
 		}
 
-		if !strings.HasSuffix(hdr.Name, "/") {
+		// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
+		// the filepath format for the OS on which the daemon is running. Hence
+		// the check for a slash-suffix MUST be done in an OS-agnostic way.
+		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
 			// Not the root directory, ensure that the parent directory exists
 			parent := filepath.Dir(hdr.Name)
 			parentPath := filepath.Join(dest, parent)
@@ -529,7 +534,7 @@ loop:
 		if err != nil {
 			return err
 		}
-		if strings.HasPrefix(rel, "../") {
+		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
 			return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
 		}
 
@@ -658,10 +663,13 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 	if err != nil {
 		return err
 	}
+
 	if srcSt.IsDir() {
 		return fmt.Errorf("Can't copy a directory")
 	}
-	// Clean up the trailing slash
+
+	// Clean up the trailing slash. This must be done in an operating
+	// system specific manner.
 	if dst[len(dst)-1] == os.PathSeparator {
 		dst = filepath.Join(dst, filepath.Base(src))
 	}
@@ -709,8 +717,10 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 // for a single file. It copies a regular file from path `src` to
 // path `dst`, and preserves all its metadata.
 //
-// If `dst` ends with a trailing slash '/', the final destination path
+// Destination handling is in an operating specific manner depending
-// will be `dst/base(src)`.
+// where the daemon is running. If `dst` ends with a trailing slash
+// the final destination path will be `dst/base(src)`  (Linux) or
+// `dst\base(src)` (Windows).
 func CopyFileWithTar(src, dst string) (err error) {
 	return defaultArchiver.CopyFileWithTar(src, dst)
 }

archive/changes.go

@@ -84,15 +84,17 @@ func Changes(layers []string, rw string) ([]Change, error) {
 		if err != nil {
 			return err
 		}
-		path = filepath.Join("/", path)
+
+		// As this runs on the daemon side, file paths are OS specific.
+		path = filepath.Join(string(os.PathSeparator), path)
 
 		// Skip root
-		if path == "/" {
+		if path == string(os.PathSeparator) {
 			return nil
 		}
 
 		// Skip AUFS metadata
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
+		if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched {
 			return err
 		}
 
@@ -169,12 +171,13 @@ type FileInfo struct {
 }
 
 func (root *FileInfo) LookUp(path string) *FileInfo {
+	// As this runs on the daemon side, file paths are OS specific.
 	parent := root
-	if path == "/" {
+	if path == string(os.PathSeparator) {
 		return root
 	}
 
-	pathElements := strings.Split(path, "/")
+	pathElements := strings.Split(path, string(os.PathSeparator))
 	for _, elem := range pathElements {
 		if elem != "" {
 			child := parent.children[elem]
@@ -189,7 +192,8 @@ func (root *FileInfo) LookUp(path string) *FileInfo {
 
 func (info *FileInfo) path() string {
 	if info.parent == nil {
-		return "/"
+		// As this runs on the daemon side, file paths are OS specific.
+		return string(os.PathSeparator)
 	}
 	return filepath.Join(info.parent.path(), info.name)
 }
@@ -257,7 +261,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
 
 	// If there were changes inside this directory, we need to add it, even if the directory
 	// itself wasn't changed. This is needed to properly save and restore filesystem permissions.
-	if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" {
+	// As this runs on the daemon side, file paths are OS specific.
+	if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) {
 		change := Change{
 			Path: info.path(),
 			Kind: ChangeModify,
@@ -279,8 +284,9 @@ func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
 }
 
 func newRootFileInfo() *FileInfo {
+	// As this runs on the daemon side, file paths are OS specific.
 	root := &FileInfo{
-		name:     "/",
+		name:     string(os.PathSeparator),
 		children: make(map[string]*FileInfo),
 	}
 	return root

archive/changes_other.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
+	"strings"
 
 	"github.com/docker/docker/pkg/system"
 )
@@ -48,9 +50,20 @@ func collectFileInfo(sourceDir string) (*FileInfo, error) {
 		if err != nil {
 			return err
 		}
-		relPath = filepath.Join("/", relPath)
 
-		if relPath == "/" {
+		// As this runs on the daemon side, file paths are OS specific.
+		relPath = filepath.Join(string(os.PathSeparator), relPath)
+
+		// See https://github.com/golang/go/issues/9168 - bug in filepath.Join.
+		// Temporary workaround. If the returned path starts with two backslashes,
+		// trim it down to a single backslash. Only relevant on Windows.
+		if runtime.GOOS == "windows" {
+			if strings.HasPrefix(relPath, `\\`) {
+				relPath = relPath[1:]
+			}
+		}
+
+		if relPath == string(os.PathSeparator) {
 			return nil
 		}
 

archive/diff.go

@@ -7,9 +7,11 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"syscall"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/pools"
 	"github.com/docker/docker/pkg/system"
 )
@@ -40,12 +42,35 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
 		// Normalize name, for safety and for a simple is-root check
 		hdr.Name = filepath.Clean(hdr.Name)
 
-		if !strings.HasSuffix(hdr.Name, "/") {
+		// Windows does not support filenames with colons in them. Ignore
+		// these files. This is not a problem though (although it might
+		// appear that it is). Let's suppose a client is running docker pull.
+		// The daemon it points to is Windows. Would it make sense for the
+		// client to be doing a docker pull Ubuntu for example (which has files
+		// with colons in the name under /usr/share/man/man3)? No, absolutely
+		// not as it would really only make sense that they were pulling a
+		// Windows image. However, for development, it is necessary to be able
+		// to pull Linux images which are in the repository.
+		//
+		// TODO Windows. Once the registry is aware of what images are Windows-
+		// specific or Linux-specific, this warning should be changed to an error
+		// to cater for the situation where someone does manage to upload a Linux
+		// image but have it tagged as Windows inadvertantly.
+		if runtime.GOOS == "windows" {
+			if strings.Contains(hdr.Name, ":") {
+				logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
+				continue
+			}
+		}
+
+		// Note as these operations are platform specific, so must the slash be.
+		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
 			// Not the root directory, ensure that the parent directory exists.
 			// This happened in some tests where an image had a tarfile without any
 			// parent directories.
 			parent := filepath.Dir(hdr.Name)
 			parentPath := filepath.Join(dest, parent)
+
 			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
 				err = system.MkdirAll(parentPath, 0600)
 				if err != nil {
@@ -74,13 +99,14 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
 			}
 			continue
 		}
-
 		path := filepath.Join(dest, hdr.Name)
 		rel, err := filepath.Rel(dest, path)
 		if err != nil {
 			return 0, err
 		}
-		if strings.HasPrefix(rel, "../") {
+
+		// Note as these operations are platform specific, so must the slash be.
+		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
 			return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
 		}
 		base := filepath.Base(path)

chrootarchive/archive.go

@@ -1,68 +1,23 @@
 package chrootarchive
 
 import (
-	"bytes"
-	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/system"
 )
 
 var chrootArchiver = &archive.Archiver{Untar: Untar}
 
-func untar() {
-	runtime.LockOSThread()
-	flag.Parse()
-
-	var options *archive.TarOptions
-
-	if runtime.GOOS != "windows" {
-		//read the options from the pipe "ExtraFiles"
-		if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
-			fatal(err)
-		}
-	} else {
-		if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
-			fatal(err)
-		}
-	}
-
-	if err := chroot(flag.Arg(0)); err != nil {
-		fatal(err)
-	}
-
-	// Explanation of Windows difference. Windows does not support chroot.
-	// untar() is a helper function for the command line in the format
-	// "docker docker-untar directory input". In Windows, directory will be
-	// something like <pathto>\docker-buildnnnnnnnnn. So, just use that directory
-	// directly instead.
-	//
-	// One example of where this is used is in the docker build command where the
-	// dockerfile will be unpacked to the machine on which the daemon runs.
-	rootPath := "/"
-	if runtime.GOOS == "windows" {
-		rootPath = flag.Arg(0)
-	}
-	if err := archive.Unpack(os.Stdin, rootPath, options); err != nil {
-		fatal(err)
-	}
-	// fully consume stdin in case it is zero padded
-	flush(os.Stdin)
-	os.Exit(0)
-}
-
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
 // and unpacks it into the directory at `dest`.
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+
 	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
 	}
@@ -84,67 +39,9 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
 	if err != nil {
 		return err
 	}
-
-	var data []byte
-	var r, w *os.File
 	defer decompressedArchive.Close()
 
-	if runtime.GOOS != "windows" {
+	return invokeUnpack(decompressedArchive, dest, options)
-		// We can't pass a potentially large exclude list directly via cmd line
-		// because we easily overrun the kernel's max argument/environment size
-		// when the full image list is passed (e.g. when this is used by
-		// `docker load`). We will marshall the options via a pipe to the
-		// child
-
-		// This solution won't work on Windows as it will fail in golang
-		// exec_windows.go as at the lowest layer because attr.Files > 3
-		r, w, err = os.Pipe()
-		if err != nil {
-			return fmt.Errorf("Untar pipe failure: %v", err)
-		}
-	} else {
-		// We can't pass the exclude list directly via cmd line
-		// because we easily overrun the shell max argument list length
-		// when the full image list is passed (e.g. when this is used
-		// by `docker load`). Instead we will add the JSON marshalled
-		// and placed in the env, which has significantly larger
-		// max size
-		data, err = json.Marshal(options)
-		if err != nil {
-			return fmt.Errorf("Untar json encode: %v", err)
-		}
-	}
-
-	cmd := reexec.Command("docker-untar", dest)
-	cmd.Stdin = decompressedArchive
-
-	if runtime.GOOS != "windows" {
-		cmd.ExtraFiles = append(cmd.ExtraFiles, r)
-		output := bytes.NewBuffer(nil)
-		cmd.Stdout = output
-		cmd.Stderr = output
-
-		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("Untar error on re-exec cmd: %v", err)
-		}
-		//write the options to the pipe for the untar exec to read
-		if err := json.NewEncoder(w).Encode(options); err != nil {
-			return fmt.Errorf("Untar json encode to pipe failed: %v", err)
-		}
-		w.Close()
-
-		if err := cmd.Wait(); err != nil {
-			return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
-		}
-		return nil
-	}
-	cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("Untar %s %s", err, out)
-	}
-	return nil
-
 }
 
 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
@@ -165,8 +62,8 @@ func CopyWithTar(src, dst string) error {
 // for a single file. It copies a regular file from path `src` to
 // path `dst`, and preserves all its metadata.
 //
-// If `dst` ends with a trailing slash '/', the final destination path
+// If `dst` ends with a trailing slash '/' ('\' on Windows), the final
-// will be `dst/base(src)`.
+// destination path will be `dst/base(src)` or `dst\base(src)`
 func CopyFileWithTar(src, dst string) (err error) {
 	return chrootArchiver.CopyFileWithTar(src, dst)
 }

chrootarchive/archive_unix.go

@@ -3,7 +3,17 @@
 package chrootarchive
 
 import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
 	"syscall"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/reexec"
 )
 
 func chroot(path string) error {
@@ -12,3 +22,64 @@ func chroot(path string) error {
 	}
 	return syscall.Chdir("/")
 }
+
+// untar is the entry-point for docker-untar on re-exec. This is not used on
+// Windows as it does not support chroot, hence no point sandboxing through
+// chroot and rexec.
+func untar() {
+	runtime.LockOSThread()
+	flag.Parse()
+
+	var options *archive.TarOptions
+
+	//read the options from the pipe "ExtraFiles"
+	if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
+		fatal(err)
+	}
+
+	if err := chroot(flag.Arg(0)); err != nil {
+		fatal(err)
+	}
+
+	if err := archive.Unpack(os.Stdin, "/", options); err != nil {
+		fatal(err)
+	}
+	// fully consume stdin in case it is zero padded
+	flush(os.Stdin)
+	os.Exit(0)
+}
+
+func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
+
+	// We can't pass a potentially large exclude list directly via cmd line
+	// because we easily overrun the kernel's max argument/environment size
+	// when the full image list is passed (e.g. when this is used by
+	// `docker load`). We will marshall the options via a pipe to the
+	// child
+	r, w, err := os.Pipe()
+	if err != nil {
+		return fmt.Errorf("Untar pipe failure: %v", err)
+	}
+
+	cmd := reexec.Command("docker-untar", dest)
+	cmd.Stdin = decompressedArchive
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
+	output := bytes.NewBuffer(nil)
+	cmd.Stdout = output
+	cmd.Stderr = output
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
+	}
+	//write the options to the pipe for the untar exec to read
+	if err := json.NewEncoder(w).Encode(options); err != nil {
+		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
+	}
+	w.Close()
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
+	}
+	return nil
+}

chrootarchive/archive_windows.go

@@ -1,6 +1,21 @@
 package chrootarchive
 
+import (
+	"io"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
 // chroot is not supported by Windows
 func chroot(path string) error {
 	return nil
 }
+
+func invokeUnpack(decompressedArchive io.ReadCloser,
+	dest string,
+	options *archive.TarOptions) error {
+	// Windows is different to Linux here because Windows does not support
+	// chroot. Hence there is no point sandboxing a chrooted process to
+	// do the unpack. We call inline instead within the daemon process.
+	return archive.Unpack(decompressedArchive, dest, options)
+}

chrootarchive/diff_unix.go

@@ -1,3 +1,5 @@
+//+build !windows
+
 package chrootarchive
 
 import (
@@ -19,41 +21,35 @@ type applyLayerResponse struct {
 	LayerSize int64 `json:"layerSize"`
 }
 
+// applyLayer is the entry-point for docker-applylayer on re-exec. This is not
+// used on Windows as it does not support chroot, hence no point sandboxing
+// through chroot and rexec.
 func applyLayer() {
 
 	var (
-		root   = "/"
 		tmpDir = ""
 		err    error
 	)
-
 	runtime.LockOSThread()
 	flag.Parse()
 
-	if runtime.GOOS != "windows" {
+	if err := chroot(flag.Arg(0)); err != nil {
-		if err := chroot(flag.Arg(0)); err != nil {
+		fatal(err)
-			fatal(err)
-		}
-
-		// We need to be able to set any perms
-		oldmask, err := system.Umask(0)
-		defer system.Umask(oldmask)
-		if err != nil {
-			fatal(err)
-		}
-	} else {
-		// As Windows does not support chroot or umask, we use the directory
-		// passed in which will be <pathto>\docker-buildnnnnnnnn instead of
-		// the 'chroot-root', "/"
-		root = flag.Arg(0)
 	}
 
-	if tmpDir, err = ioutil.TempDir(root, "temp-docker-extract"); err != nil {
+	// We need to be able to set any perms
+	oldmask, err := system.Umask(0)
+	defer system.Umask(oldmask)
+	if err != nil {
+		fatal(err)
+	}
+
+	if tmpDir, err = ioutil.TempDir("/", "temp-docker-extract"); err != nil {
 		fatal(err)
 	}
 
 	os.Setenv("TMPDIR", tmpDir)
-	size, err := archive.UnpackLayer(root, os.Stdin)
+	size, err := archive.UnpackLayer("/", os.Stdin)
 	os.RemoveAll(tmpDir)
 	if err != nil {
 		fatal(err)

chrootarchive/diff_windows.go

@@ -0,0 +1,32 @@
+package chrootarchive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
+	dest = filepath.Clean(dest)
+	decompressed, err := archive.DecompressStream(layer)
+	if err != nil {
+		return 0, err
+	}
+	defer decompressed.Close()
+
+	tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract")
+	if err != nil {
+		return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err)
+	}
+
+	s, err := archive.UnpackLayer(dest, decompressed)
+	os.RemoveAll(tmpDir)
+	if err != nil {
+		return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest)
+	}
+
+	return s, nil
+}

chrootarchive/init_unix.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package chrootarchive
 
 import (
@@ -10,8 +12,8 @@ import (
 )
 
 func init() {
-	reexec.Register("docker-untar", untar)
 	reexec.Register("docker-applyLayer", applyLayer)
+	reexec.Register("docker-untar", untar)
 }
 
 func fatal(err error) {

chrootarchive/init_windows.go

@@ -0,0 +1,4 @@
+package chrootarchive
+
+func init() {
+}