vbatts/pkg
1
0
Fork 0
Code Pull requests Releases Activity

Windows: Docker build starting to work

Browse source 
Signed-off-by: John Howard <jhoward@microsoft.com>
This commit is contained in:
John Howard 2015-06-01 16:42:27 -07:00
parent bfdf63d37c
commit 65cbc7cb25
11 changed files with 219 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
archive/archive.go
View file

@ -452,6 +452,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
}
seen[relFilePath] = true
// TODO Windows: Verify if this needs to be os.Pathseparator
// Rename the base resource
if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) {
renamedRelFilePath = relFilePath
@ -503,7 +504,8 @@ loop:
}
// Normalize name, for safety and for a simple is-root check
// This keeps "../" as-is, but normalizes "/../" to "/"
// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
// This keeps "..\" as-is, but normalizes "\..\" to "\".
hdr.Name = filepath.Clean(hdr.Name)
for _, exclude := range options.ExcludePatterns {
@ -512,7 +514,10 @@ loop:
}
}
if !strings.HasSuffix(hdr.Name, "/") {
// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
// the filepath format for the OS on which the daemon is running. Hence
// the check for a slash-suffix MUST be done in an OS-agnostic way.
if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
// Not the root directory, ensure that the parent directory exists
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
@ -529,7 +534,7 @@ loop:
if err != nil {
return err
}
if strings.HasPrefix(rel, "../") {
if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
@ -658,10 +663,13 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
if err != nil {
return err
}
if srcSt.IsDir() {
return fmt.Errorf("Can't copy a directory")
}
// Clean up the trailing slash
// Clean up the trailing slash. This must be done in an operating
// system specific manner.
if dst[len(dst)-1] == os.PathSeparator {
dst = filepath.Join(dst, filepath.Base(src))
}
@ -709,8 +717,10 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
// Destination handling is in an operating specific manner depending
// where the daemon is running. If `dst` ends with a trailing slash
// the final destination path will be `dst/base(src)` (Linux) or
// `dst\base(src)` (Windows).
func CopyFileWithTar(src, dst string) (err error) {
return defaultArchiver.CopyFileWithTar(src, dst)
}

22
archive/changes.go
View file

@ -84,15 +84,17 @@ func Changes(layers []string, rw string) ([]Change, error) {
if err != nil {
return err
}
path = filepath.Join("/", path)
// As this runs on the daemon side, file paths are OS specific.
path = filepath.Join(string(os.PathSeparator), path)
// Skip root
if path == "/" {
if path == string(os.PathSeparator) {
return nil
}
// Skip AUFS metadata
if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
if matched, err := filepath.Match(string(os.PathSeparator)+".wh..wh.*", path); err != nil || matched {
return err
}
@ -169,12 +171,13 @@ type FileInfo struct {
}
func (root *FileInfo) LookUp(path string) *FileInfo {
// As this runs on the daemon side, file paths are OS specific.
parent := root
if path == "/" {
if path == string(os.PathSeparator) {
return root
}
pathElements := strings.Split(path, "/")
pathElements := strings.Split(path, string(os.PathSeparator))
for _, elem := range pathElements {
if elem != "" {
child := parent.children[elem]
@ -189,7 +192,8 @@ func (root *FileInfo) LookUp(path string) *FileInfo {
func (info *FileInfo) path() string {
if info.parent == nil {
return "/"
// As this runs on the daemon side, file paths are OS specific.
return string(os.PathSeparator)
}
return filepath.Join(info.parent.path(), info.name)
}
@ -257,7 +261,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
// If there were changes inside this directory, we need to add it, even if the directory
// itself wasn't changed. This is needed to properly save and restore filesystem permissions.
if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" {
// As this runs on the daemon side, file paths are OS specific.
if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) {
change := Change{
Path: info.path(),
Kind: ChangeModify,
@ -279,8 +284,9 @@ func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
}
func newRootFileInfo() *FileInfo {
// As this runs on the daemon side, file paths are OS specific.
root := &FileInfo{
name: "/",
name: string(os.PathSeparator),
children: make(map[string]*FileInfo),
}
return root

17
archive/changes_other.go
View file

@ -6,6 +6,8 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/docker/docker/pkg/system"
)
@ -48,9 +50,20 @@ func collectFileInfo(sourceDir string) (*FileInfo, error) {
if err != nil {
return err
}
relPath = filepath.Join("/", relPath)
if relPath == "/" {
// As this runs on the daemon side, file paths are OS specific.
relPath = filepath.Join(string(os.PathSeparator), relPath)
// See https://github.com/golang/go/issues/9168 - bug in filepath.Join.
// Temporary workaround. If the returned path starts with two backslashes,
// trim it down to a single backslash. Only relevant on Windows.
if runtime.GOOS == "windows" {
if strings.HasPrefix(relPath, `\\`) {
relPath = relPath[1:]
}
}
if relPath == string(os.PathSeparator) {
return nil
}

32
archive/diff.go
View file

@ -7,9 +7,11 @@ import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/pools"
"github.com/docker/docker/pkg/system"
)
@ -40,12 +42,35 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
// Normalize name, for safety and for a simple is-root check
hdr.Name = filepath.Clean(hdr.Name)
if !strings.HasSuffix(hdr.Name, "/") {
// Windows does not support filenames with colons in them. Ignore
// these files. This is not a problem though (although it might
// appear that it is). Let's suppose a client is running docker pull.
// The daemon it points to is Windows. Would it make sense for the
// client to be doing a docker pull Ubuntu for example (which has files
// with colons in the name under /usr/share/man/man3)? No, absolutely
// not as it would really only make sense that they were pulling a
// Windows image. However, for development, it is necessary to be able
// to pull Linux images which are in the repository.
//
// TODO Windows. Once the registry is aware of what images are Windows-
// specific or Linux-specific, this warning should be changed to an error
// to cater for the situation where someone does manage to upload a Linux
// image but have it tagged as Windows inadvertantly.
if runtime.GOOS == "windows" {
if strings.Contains(hdr.Name, ":") {
logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
continue
}
}
// Note as these operations are platform specific, so must the slash be.
if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
// Not the root directory, ensure that the parent directory exists.
// This happened in some tests where an image had a tarfile without any
// parent directories.
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
err = system.MkdirAll(parentPath, 0600)
if err != nil {
@ -74,13 +99,14 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
}
continue
}
path := filepath.Join(dest, hdr.Name)
rel, err := filepath.Rel(dest, path)
if err != nil {
return 0, err
}
if strings.HasPrefix(rel, "../") {
// Note as these operations are platform specific, so must the slash be.
if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
base := filepath.Base(path)

111
chrootarchive/archive.go
View file

@ -1,68 +1,23 @@
package chrootarchive
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/docker/docker/pkg/system"
)
var chrootArchiver = &archive.Archiver{Untar: Untar}
func untar() {
runtime.LockOSThread()
flag.Parse()
var options *archive.TarOptions
if runtime.GOOS != "windows" {
//read the options from the pipe "ExtraFiles"
if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
fatal(err)
}
} else {
if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
fatal(err)
}
}
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
// Explanation of Windows difference. Windows does not support chroot.
// untar() is a helper function for the command line in the format
// "docker docker-untar directory input". In Windows, directory will be
// something like <pathto>\docker-buildnnnnnnnnn. So, just use that directory
// directly instead.
//
// One example of where this is used is in the docker build command where the
// dockerfile will be unpacked to the machine on which the daemon runs.
rootPath := "/"
if runtime.GOOS == "windows" {
rootPath = flag.Arg(0)
}
if err := archive.Unpack(os.Stdin, rootPath, options); err != nil {
fatal(err)
}
// fully consume stdin in case it is zero padded
flush(os.Stdin)
os.Exit(0)
}
// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive may be compressed with one of the following algorithms:
// identity (uncompressed), gzip, bzip2, xz.
func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
if tarArchive == nil {
return fmt.Errorf("Empty archive")
}
@ -84,67 +39,9 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
if err != nil {
return err
}
var data []byte
var r, w *os.File
defer decompressedArchive.Close()
if runtime.GOOS != "windows" {
// We can't pass a potentially large exclude list directly via cmd line
// because we easily overrun the kernel's max argument/environment size
// when the full image list is passed (e.g. when this is used by
// `docker load`). We will marshall the options via a pipe to the
// child
// This solution won't work on Windows as it will fail in golang
// exec_windows.go as at the lowest layer because attr.Files > 3
r, w, err = os.Pipe()
if err != nil {
return fmt.Errorf("Untar pipe failure: %v", err)
}
} else {
// We can't pass the exclude list directly via cmd line
// because we easily overrun the shell max argument list length
// when the full image list is passed (e.g. when this is used
// by `docker load`). Instead we will add the JSON marshalled
// and placed in the env, which has significantly larger
// max size
data, err = json.Marshal(options)
if err != nil {
return fmt.Errorf("Untar json encode: %v", err)
}
}
cmd := reexec.Command("docker-untar", dest)
cmd.Stdin = decompressedArchive
if runtime.GOOS != "windows" {
cmd.ExtraFiles = append(cmd.ExtraFiles, r)
output := bytes.NewBuffer(nil)
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Start(); err != nil {
return fmt.Errorf("Untar error on re-exec cmd: %v", err)
}
//write the options to the pipe for the untar exec to read
if err := json.NewEncoder(w).Encode(options); err != nil {
return fmt.Errorf("Untar json encode to pipe failed: %v", err)
}
w.Close()
if err := cmd.Wait(); err != nil {
return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
}
return nil
}
cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("Untar %s %s", err, out)
}
return nil
return invokeUnpack(decompressedArchive, dest, options)
}
// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
@ -165,8 +62,8 @@ func CopyWithTar(src, dst string) error {
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
// If `dst` ends with a trailing slash '/' ('\' on Windows), the final
// destination path will be `dst/base(src)` or `dst\base(src)`
func CopyFileWithTar(src, dst string) (err error) {
return chrootArchiver.CopyFileWithTar(src, dst)
}

71
chrootarchive/archive_unix.go
View file

@ -3,7 +3,17 @@
package chrootarchive
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"runtime"
"syscall"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
)
func chroot(path string) error {
@ -12,3 +22,64 @@ func chroot(path string) error {
}
return syscall.Chdir("/")
}
// untar is the entry-point for docker-untar on re-exec. This is not used on
// Windows as it does not support chroot, hence no point sandboxing through
// chroot and rexec.
func untar() {
runtime.LockOSThread()
flag.Parse()
var options *archive.TarOptions
//read the options from the pipe "ExtraFiles"
if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
fatal(err)
}
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
if err := archive.Unpack(os.Stdin, "/", options); err != nil {
fatal(err)
}
// fully consume stdin in case it is zero padded
flush(os.Stdin)
os.Exit(0)
}
func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
// We can't pass a potentially large exclude list directly via cmd line
// because we easily overrun the kernel's max argument/environment size
// when the full image list is passed (e.g. when this is used by
// `docker load`). We will marshall the options via a pipe to the
// child
r, w, err := os.Pipe()
if err != nil {
return fmt.Errorf("Untar pipe failure: %v", err)
}
cmd := reexec.Command("docker-untar", dest)
cmd.Stdin = decompressedArchive
cmd.ExtraFiles = append(cmd.ExtraFiles, r)
output := bytes.NewBuffer(nil)
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Start(); err != nil {
return fmt.Errorf("Untar error on re-exec cmd: %v", err)
}
//write the options to the pipe for the untar exec to read
if err := json.NewEncoder(w).Encode(options); err != nil {
return fmt.Errorf("Untar json encode to pipe failed: %v", err)
}
w.Close()
if err := cmd.Wait(); err != nil {
return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
}
return nil
}

15
chrootarchive/archive_windows.go
View file

@ -1,6 +1,21 @@
package chrootarchive
import (
"io"
"github.com/docker/docker/pkg/archive"
)
// chroot is not supported by Windows
func chroot(path string) error {
return nil
}
func invokeUnpack(decompressedArchive io.ReadCloser,
dest string,
options *archive.TarOptions) error {
// Windows is different to Linux here because Windows does not support
// chroot. Hence there is no point sandboxing a chrooted process to
// do the unpack. We call inline instead within the daemon process.
return archive.Unpack(decompressedArchive, dest, options)
}

18
chrootarchive/diff.go → chrootarchive/diff_unix.go
View file

@ -1,3 +1,5 @@
//+build !windows
package chrootarchive
import (
@ -19,18 +21,18 @@ type applyLayerResponse struct {
LayerSize int64 `json:"layerSize"`
}
// applyLayer is the entry-point for docker-applylayer on re-exec. This is not
// used on Windows as it does not support chroot, hence no point sandboxing
// through chroot and rexec.
func applyLayer() {
var (
root = "/"
tmpDir = ""
err error
)
runtime.LockOSThread()
flag.Parse()
if runtime.GOOS != "windows" {
if err := chroot(flag.Arg(0)); err != nil {
fatal(err)
}
@ -41,19 +43,13 @@ func applyLayer() {
if err != nil {
fatal(err)
}
} else {
// As Windows does not support chroot or umask, we use the directory
// passed in which will be <pathto>\docker-buildnnnnnnnn instead of
// the 'chroot-root', "/"
root = flag.Arg(0)
}
if tmpDir, err = ioutil.TempDir(root, "temp-docker-extract"); err != nil {
if tmpDir, err = ioutil.TempDir("/", "temp-docker-extract"); err != nil {
fatal(err)
}
os.Setenv("TMPDIR", tmpDir)
size, err := archive.UnpackLayer(root, os.Stdin)
size, err := archive.UnpackLayer("/", os.Stdin)
os.RemoveAll(tmpDir)
if err != nil {
fatal(err)

32
chrootarchive/diff_windows.go Normal file
View file

@ -0,0 +1,32 @@
package chrootarchive
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/docker/docker/pkg/archive"
)
func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
dest = filepath.Clean(dest)
decompressed, err := archive.DecompressStream(layer)
if err != nil {
return 0, err
}
defer decompressed.Close()
tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract")
if err != nil {
return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err)
}
s, err := archive.UnpackLayer(dest, decompressed)
os.RemoveAll(tmpDir)
if err != nil {
return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest)
}
return s, nil
}

4
chrootarchive/init.go → chrootarchive/init_unix.go
View file

@ -1,3 +1,5 @@
// +build !windows
package chrootarchive
import (
@ -10,8 +12,8 @@ import (
)
func init() {
reexec.Register("docker-untar", untar)
reexec.Register("docker-applyLayer", applyLayer)
reexec.Register("docker-untar", untar)
}
func fatal(err error) {

4
chrootarchive/init_windows.go Normal file
View file

@ -0,0 +1,4 @@
package chrootarchive
func init() {
}