Windows: Docker build starting to work

Signed-off-by: John Howard <jhoward@microsoft.com>

chrootarchive/archive.go

@@ -1,68 +1,23 @@
 package chrootarchive
 
 import (
-	"bytes"
-	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/system"
 )
 
 var chrootArchiver = &archive.Archiver{Untar: Untar}
 
-func untar() {
-	runtime.LockOSThread()
-	flag.Parse()
-
-	var options *archive.TarOptions
-
-	if runtime.GOOS != "windows" {
-		//read the options from the pipe "ExtraFiles"
-		if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
-			fatal(err)
-		}
-	} else {
-		if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
-			fatal(err)
-		}
-	}
-
-	if err := chroot(flag.Arg(0)); err != nil {
-		fatal(err)
-	}
-
-	// Explanation of Windows difference. Windows does not support chroot.
-	// untar() is a helper function for the command line in the format
-	// "docker docker-untar directory input". In Windows, directory will be
-	// something like <pathto>\docker-buildnnnnnnnnn. So, just use that directory
-	// directly instead.
-	//
-	// One example of where this is used is in the docker build command where the
-	// dockerfile will be unpacked to the machine on which the daemon runs.
-	rootPath := "/"
-	if runtime.GOOS == "windows" {
-		rootPath = flag.Arg(0)
-	}
-	if err := archive.Unpack(os.Stdin, rootPath, options); err != nil {
-		fatal(err)
-	}
-	// fully consume stdin in case it is zero padded
-	flush(os.Stdin)
-	os.Exit(0)
-}
-
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
 // and unpacks it into the directory at `dest`.
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+
 	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
 	}
@@ -84,67 +39,9 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
 	if err != nil {
 		return err
 	}
-
-	var data []byte
-	var r, w *os.File
 	defer decompressedArchive.Close()
 
-	if runtime.GOOS != "windows" {
-		// We can't pass a potentially large exclude list directly via cmd line
-		// because we easily overrun the kernel's max argument/environment size
-		// when the full image list is passed (e.g. when this is used by
-		// `docker load`). We will marshall the options via a pipe to the
-		// child
-
-		// This solution won't work on Windows as it will fail in golang
-		// exec_windows.go as at the lowest layer because attr.Files > 3
-		r, w, err = os.Pipe()
-		if err != nil {
-			return fmt.Errorf("Untar pipe failure: %v", err)
-		}
-	} else {
-		// We can't pass the exclude list directly via cmd line
-		// because we easily overrun the shell max argument list length
-		// when the full image list is passed (e.g. when this is used
-		// by `docker load`). Instead we will add the JSON marshalled
-		// and placed in the env, which has significantly larger
-		// max size
-		data, err = json.Marshal(options)
-		if err != nil {
-			return fmt.Errorf("Untar json encode: %v", err)
-		}
-	}
-
-	cmd := reexec.Command("docker-untar", dest)
-	cmd.Stdin = decompressedArchive
-
-	if runtime.GOOS != "windows" {
-		cmd.ExtraFiles = append(cmd.ExtraFiles, r)
-		output := bytes.NewBuffer(nil)
-		cmd.Stdout = output
-		cmd.Stderr = output
-
-		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("Untar error on re-exec cmd: %v", err)
-		}
-		//write the options to the pipe for the untar exec to read
-		if err := json.NewEncoder(w).Encode(options); err != nil {
-			return fmt.Errorf("Untar json encode to pipe failed: %v", err)
-		}
-		w.Close()
-
-		if err := cmd.Wait(); err != nil {
-			return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
-		}
-		return nil
-	}
-	cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("Untar %s %s", err, out)
-	}
-	return nil
-
+	return invokeUnpack(decompressedArchive, dest, options)
 }
 
 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
@@ -165,8 +62,8 @@ func CopyWithTar(src, dst string) error {
 // for a single file. It copies a regular file from path `src` to
 // path `dst`, and preserves all its metadata.
 //
-// If `dst` ends with a trailing slash '/', the final destination path
-// will be `dst/base(src)`.
+// If `dst` ends with a trailing slash '/' ('\' on Windows), the final
+// destination path will be `dst/base(src)` or `dst\base(src)`
 func CopyFileWithTar(src, dst string) (err error) {
 	return chrootArchiver.CopyFileWithTar(src, dst)
 }

chrootarchive/archive_unix.go

@@ -3,7 +3,17 @@
 package chrootarchive
 
 import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
 	"syscall"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/reexec"
 )
 
 func chroot(path string) error {
@@ -12,3 +22,64 @@ func chroot(path string) error {
 	}
 	return syscall.Chdir("/")
 }
+
+// untar is the entry-point for docker-untar on re-exec. This is not used on
+// Windows as it does not support chroot, hence no point sandboxing through
+// chroot and rexec.
+func untar() {
+	runtime.LockOSThread()
+	flag.Parse()
+
+	var options *archive.TarOptions
+
+	//read the options from the pipe "ExtraFiles"
+	if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
+		fatal(err)
+	}
+
+	if err := chroot(flag.Arg(0)); err != nil {
+		fatal(err)
+	}
+
+	if err := archive.Unpack(os.Stdin, "/", options); err != nil {
+		fatal(err)
+	}
+	// fully consume stdin in case it is zero padded
+	flush(os.Stdin)
+	os.Exit(0)
+}
+
+func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
+
+	// We can't pass a potentially large exclude list directly via cmd line
+	// because we easily overrun the kernel's max argument/environment size
+	// when the full image list is passed (e.g. when this is used by
+	// `docker load`). We will marshall the options via a pipe to the
+	// child
+	r, w, err := os.Pipe()
+	if err != nil {
+		return fmt.Errorf("Untar pipe failure: %v", err)
+	}
+
+	cmd := reexec.Command("docker-untar", dest)
+	cmd.Stdin = decompressedArchive
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
+	output := bytes.NewBuffer(nil)
+	cmd.Stdout = output
+	cmd.Stderr = output
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
+	}
+	//write the options to the pipe for the untar exec to read
+	if err := json.NewEncoder(w).Encode(options); err != nil {
+		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
+	}
+	w.Close()
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
+	}
+	return nil
+}

chrootarchive/archive_windows.go

@@ -1,6 +1,21 @@
 package chrootarchive
 
+import (
+	"io"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
 // chroot is not supported by Windows
 func chroot(path string) error {
 	return nil
 }
+
+func invokeUnpack(decompressedArchive io.ReadCloser,
+	dest string,
+	options *archive.TarOptions) error {
+	// Windows is different to Linux here because Windows does not support
+	// chroot. Hence there is no point sandboxing a chrooted process to
+	// do the unpack. We call inline instead within the daemon process.
+	return archive.Unpack(decompressedArchive, dest, options)
+}

chrootarchive/diff_unix.go

@@ -1,3 +1,5 @@
+//+build !windows
+
 package chrootarchive
 
 import (
@@ -19,41 +21,35 @@ type applyLayerResponse struct {
 	LayerSize int64 `json:"layerSize"`
 }
 
+// applyLayer is the entry-point for docker-applylayer on re-exec. This is not
+// used on Windows as it does not support chroot, hence no point sandboxing
+// through chroot and rexec.
 func applyLayer() {
 
 	var (
-		root   = "/"
 		tmpDir = ""
 		err    error
 	)
-
 	runtime.LockOSThread()
 	flag.Parse()
 
-	if runtime.GOOS != "windows" {
-		if err := chroot(flag.Arg(0)); err != nil {
-			fatal(err)
-		}
-
-		// We need to be able to set any perms
-		oldmask, err := system.Umask(0)
-		defer system.Umask(oldmask)
-		if err != nil {
-			fatal(err)
-		}
-	} else {
-		// As Windows does not support chroot or umask, we use the directory
-		// passed in which will be <pathto>\docker-buildnnnnnnnn instead of
-		// the 'chroot-root', "/"
-		root = flag.Arg(0)
+	if err := chroot(flag.Arg(0)); err != nil {
+		fatal(err)
 	}
 
-	if tmpDir, err = ioutil.TempDir(root, "temp-docker-extract"); err != nil {
+	// We need to be able to set any perms
+	oldmask, err := system.Umask(0)
+	defer system.Umask(oldmask)
+	if err != nil {
+		fatal(err)
+	}
+
+	if tmpDir, err = ioutil.TempDir("/", "temp-docker-extract"); err != nil {
 		fatal(err)
 	}
 
 	os.Setenv("TMPDIR", tmpDir)
-	size, err := archive.UnpackLayer(root, os.Stdin)
+	size, err := archive.UnpackLayer("/", os.Stdin)
 	os.RemoveAll(tmpDir)
 	if err != nil {
 		fatal(err)

chrootarchive/diff_windows.go

@@ -0,0 +1,32 @@
+package chrootarchive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
+	dest = filepath.Clean(dest)
+	decompressed, err := archive.DecompressStream(layer)
+	if err != nil {
+		return 0, err
+	}
+	defer decompressed.Close()
+
+	tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract")
+	if err != nil {
+		return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err)
+	}
+
+	s, err := archive.UnpackLayer(dest, decompressed)
+	os.RemoveAll(tmpDir)
+	if err != nil {
+		return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest)
+	}
+
+	return s, nil
+}

chrootarchive/init_unix.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package chrootarchive
 
 import (
@@ -10,8 +12,8 @@ import (
 )
 
 func init() {
-	reexec.Register("docker-untar", untar)
 	reexec.Register("docker-applyLayer", applyLayer)
+	reexec.Register("docker-untar", untar)
 }
 
 func fatal(err error) {

chrootarchive/init_windows.go

@@ -0,0 +1,4 @@
+package chrootarchive
+
+func init() {
+}