containerd/windows/hcs/hcs.go

// +build windows

package hcs

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Microsoft/hcsshim"
	"github.com/Sirupsen/logrus"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/log"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
)

const (
	layerFile               = "layer"
	defaultTerminateTimeout = 5 * time.Minute
)

func LoadAll(ctx context.Context, owner, rootDir string) (map[string]*HCS, error) {
	ctrProps, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{})
	if err != nil {
		return nil, errors.Wrap(err, "failed to retrieve running containers")
	}

	containers := make(map[string]*HCS)
	for _, p := range ctrProps {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}

		if p.Owner != owner || p.SystemType != "Container" {
			continue
		}

		// TODO: take context in account
		container, err := hcsshim.OpenContainer(p.ID)
		if err != nil {
			return nil, errors.Wrapf(err, "failed open container %s", p.ID)
		}
		stateDir := filepath.Join(rootDir, p.ID)
		b, err := ioutil.ReadFile(filepath.Join(stateDir, layerFile))
		containers[p.ID] = &HCS{
			id:              p.ID,
			container:       container,
			stateDir:        stateDir,
			layerFolderPath: string(b),
			conf: Configuration{
				TerminateDuration: defaultTerminateTimeout,
			},
		}
	}

	return containers, nil
}

// New creates a new container (but doesn't start) it.
func New(rootDir, owner, containerID string, spec specs.Spec, conf Configuration, cio containerd.IO) (*HCS, error) {
	stateDir := filepath.Join(rootDir, containerID)
	if err := os.MkdirAll(stateDir, 0755); err != nil {
		return nil, errors.Wrapf(err, "unable to create container state dir %s", stateDir)
	}

	if conf.TerminateDuration == 0 {
		conf.TerminateDuration = defaultTerminateTimeout
	}

	h := &HCS{
		stateDir: stateDir,
		owner:    owner,
		id:       containerID,
		spec:     spec,
		conf:     conf,
	}

	sio, err := newSIO(cio)
	if err != nil {
		return nil, err
	}
	h.io = sio
	runtime.SetFinalizer(sio, func(s *shimIO) {
		s.Close()
	})

	hcsConf, err := h.newHCSConfiguration()
	if err != nil {
		return nil, err
	}

	ctr, err := hcsshim.CreateContainer(containerID, hcsConf)
	if err != nil {
		removeLayer(context.TODO(), hcsConf.LayerFolderPath)
		return nil, err
	}
	h.container = ctr
	h.layerFolderPath = hcsConf.LayerFolderPath

	return h, nil
}

type HCS struct {
	stateDir        string
	owner           string
	id              string
	spec            specs.Spec
	conf            Configuration
	io              *shimIO
	container       hcsshim.Container
	initProcess     hcsshim.Process
	layerFolderPath string
}

// Start starts the associated container and instantiate the init
// process within it.
func (s *HCS) Start(ctx context.Context, servicing bool) error {
	if s.initProcess != nil {
		return errors.New("init process already started")
	}
	if err := s.container.Start(); err != nil {
		if err := s.Terminate(ctx); err != nil {
			log.G(ctx).WithError(err).Errorf("failed to terminate container %s", s.id)
		}
		return err
	}

	proc, err := s.newProcess(ctx, s.io, s.spec.Process)
	if err != nil {
		s.Terminate(ctx)
		return err
	}

	s.initProcess = proc

	return nil
}

// Pid returns the pid of the container init process
func (s *HCS) Pid() int {
	return s.initProcess.Pid()
}

// ExitCode waits for the container to exit and return the exit code
// of the init process
func (s *HCS) ExitCode(ctx context.Context) (uint32, error) {
	// TODO: handle a context cancellation
	if err := s.initProcess.Wait(); err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			return 255, errors.Wrapf(err, "failed to wait for container '%s' init process", s.id)
		}
		// container is probably dead, let's try to get its exit code
	}

	ec, err := s.initProcess.ExitCode()
	if err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			return 255, errors.Wrapf(err, "failed to get container '%s' init process exit code", s.id)
		}
		// Well, unknown exit code it is
		ec = 255
	}

	return uint32(ec), err
}

// Exec starts a new process within the container
func (s *HCS) Exec(ctx context.Context, procSpec specs.Process, io containerd.IO) (*Process, error) {
	sio, err := newSIO(io)
	if err != nil {
		return nil, err
	}
	p, err := s.newProcess(ctx, sio, procSpec)
	if err != nil {
		return nil, err
	}

	return &Process{
		containerID: s.id,
		p:           p,
		status:      containerd.RunningStatus,
	}, nil
}

// newProcess create a new process within a running container. This is
// used to create both the init process and subsequent 'exec'
// processes.
func (s *HCS) newProcess(ctx context.Context, sio *shimIO, procSpec specs.Process) (hcsshim.Process, error) {
	conf := hcsshim.ProcessConfig{
		EmulateConsole:   sio.terminal,
		CreateStdInPipe:  sio.stdin != nil,
		CreateStdOutPipe: sio.stdout != nil,
		CreateStdErrPipe: sio.stderr != nil,
		User:             procSpec.User.Username,
		CommandLine:      strings.Join(procSpec.Args, " "),
		Environment:      ociSpecEnvToHCSEnv(procSpec.Env),
		WorkingDirectory: procSpec.Cwd,
	}
	conf.ConsoleSize[0] = procSpec.ConsoleSize.Height
	conf.ConsoleSize[1] = procSpec.ConsoleSize.Width

	if conf.WorkingDirectory == "" {
		conf.WorkingDirectory = s.spec.Process.Cwd
	}

	proc, err := s.container.CreateProcess(&conf)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create process with conf %#v", conf)

	}
	pid := proc.Pid()

	stdin, stdout, stderr, err := proc.Stdio()
	if err != nil {
		s.Terminate(ctx)
		return nil, err
	}

	if sio.stdin != nil {
		go func() {
			log.G(ctx).WithField("pid", pid).Debug("stdin: copy started")
			io.Copy(stdin, sio.stdin)
			log.G(ctx).WithField("pid", pid).Debug("stdin: copy done")
			stdin.Close()
			sio.stdin.Close()
		}()
	} else {
		proc.CloseStdin()
	}

	if sio.stdout != nil {
		go func() {
			log.G(ctx).WithField("pid", pid).Debug("stdout: copy started")
			io.Copy(sio.stdout, stdout)
			log.G(ctx).WithField("pid", pid).Debug("stdout: copy done")
			stdout.Close()
			sio.stdout.Close()
		}()
	}

	if sio.stderr != nil {
		go func() {
			log.G(ctx).WithField("pid", pid).Debug("stderr: copy started")
			io.Copy(sio.stderr, stderr)
			log.G(ctx).WithField("pid", pid).Debug("stderr: copy done")
			stderr.Close()
			sio.stderr.Close()
		}()
	}

	return proc, nil
}

// Terminate stop a running container.
func (s *HCS) Terminate(ctx context.Context) error {
	err := s.container.Terminate()
	switch {
	case hcsshim.IsPending(err):
		// TODO: take the context into account
		err = s.container.WaitTimeout(s.conf.TerminateDuration)
	case hcsshim.IsAlreadyStopped(err):
		err = nil
	}

	return err
}

func (s *HCS) Shutdown(ctx context.Context) error {
	err := s.container.Shutdown()
	switch {
	case hcsshim.IsPending(err):
		// TODO: take the context into account
		err = s.container.WaitTimeout(s.conf.TerminateDuration)
	case hcsshim.IsAlreadyStopped(err):
		err = nil
	}

	if err != nil {
		log.G(ctx).WithError(err).Debugf("failed to shutdown container %s, calling terminate", s.id)
		return s.Terminate(ctx)
	}

	return nil
}

// Remove start a servicing container if needed then cleanup the container
// resources
func (s *HCS) Remove(ctx context.Context) error {
	defer func() {
		if err := s.Shutdown(ctx); err != nil {
			log.G(ctx).WithError(err).WithField("id", s.id).
				Errorf("failed to shutdown/terminate container")
		}

		if s.initProcess != nil {
			if err := s.initProcess.Close(); err != nil {
				log.G(ctx).WithError(err).WithFields(logrus.Fields{"pid": s.Pid(), "id": s.id}).
					Errorf("failed to clean init process resources")
			}
		}
		if err := s.container.Close(); err != nil {
			log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to clean container resources")
		}

		// Cleanup folder layer
		if err := removeLayer(ctx, s.layerFolderPath); err == nil {
			os.RemoveAll(s.stateDir)
		}
	}()

	if update, err := s.container.HasPendingUpdates(); err != nil || !update {
		return nil
	}

	// TODO: take the context into account
	serviceHCS, err := New(s.stateDir, s.owner, s.id+"_servicing", s.spec, s.conf, containerd.IO{})
	if err != nil {
		log.G(ctx).WithError(err).WithField("id", s.id).Warn("could not create servicing container")
		return nil
	}
	defer serviceHCS.container.Close()

	err = serviceHCS.Start(ctx, true)
	if err != nil {
		if err := serviceHCS.Terminate(ctx); err != nil {
			log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to terminate servicing container for %s")
		}
		log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to start servicing container")
		return nil
	}

	// wait for the container to exit
	_, err = serviceHCS.ExitCode(ctx)
	if err != nil {
		if err := serviceHCS.Terminate(ctx); err != nil {
			log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to terminate servicing container for %s")
		}
		log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to get servicing container exit code")
	}

	serviceHCS.container.WaitTimeout(s.conf.TerminateDuration)

	return nil
}

// newHCSConfiguration generates a hcsshim configuration from the instance
// OCI Spec and hcs.Configuration.
func (s *HCS) newHCSConfiguration() (*hcsshim.ContainerConfig, error) {
	configuration := &hcsshim.ContainerConfig{
		SystemType:                 "Container",
		Name:                       s.id,
		Owner:                      s.owner,
		HostName:                   s.spec.Hostname,
		IgnoreFlushesDuringBoot:    s.conf.IgnoreFlushesDuringBoot,
		HvPartition:                s.conf.UseHyperV,
		AllowUnqualifiedDNSQuery:   s.conf.AllowUnqualifiedDNSQuery,
		EndpointList:               s.conf.NetworkEndpoints,
		NetworkSharedContainerName: s.conf.NetworkSharedContainerID,
		Credentials:                s.conf.Credentials,
	}

	// TODO: use the create request Mount for those
	for _, layerPath := range s.conf.Layers {
		_, filename := filepath.Split(layerPath)
		guid, err := hcsshim.NameToGuid(filename)
		if err != nil {
			return nil, err
		}
		configuration.Layers = append(configuration.Layers, hcsshim.Layer{
			ID:   guid.ToString(),
			Path: layerPath,
		})
	}

	if len(s.spec.Mounts) > 0 {
		mds := make([]hcsshim.MappedDir, len(s.spec.Mounts))
		for i, mount := range s.spec.Mounts {
			mds[i] = hcsshim.MappedDir{
				HostPath:      mount.Source,
				ContainerPath: mount.Destination,
				ReadOnly:      false,
			}
			for _, o := range mount.Options {
				if strings.ToLower(o) == "ro" {
					mds[i].ReadOnly = true
				}
			}
		}
		configuration.MappedDirectories = mds
	}

	if s.conf.DNSSearchList != nil {
		configuration.DNSSearchList = strings.Join(s.conf.DNSSearchList, ",")
	}

	if configuration.HvPartition {
		for _, layerPath := range s.conf.Layers {
			utilityVMPath := filepath.Join(layerPath, "UtilityVM")
			_, err := os.Stat(utilityVMPath)
			if err == nil {
				configuration.HvRuntime = &hcsshim.HvRuntime{ImagePath: utilityVMPath}
				break
			} else if !os.IsNotExist(err) {
				return nil, errors.Wrapf(err, "failed to access layer %s", layerPath)
			}
		}
	}

	if len(configuration.Layers) == 0 {
		// TODO: support starting with 0 layers, this mean we need the "filter" directory as parameter
		return nil, errors.New("at least one layers must be provided")
	}

	di := hcsshim.DriverInfo{
		Flavour: 1, // filter driver
	}

	if len(configuration.Layers) > 0 {
		di.HomeDir = filepath.Dir(s.conf.Layers[0])
	}

	// Windows doesn't support creating a container with a readonly
	// filesystem, so always create a RW one
	if err := hcsshim.CreateSandboxLayer(di, s.id, s.conf.Layers[0], s.conf.Layers); err != nil {
		return nil, errors.Wrapf(err, "failed to create sandbox layer for %s: layers: %#v, driverInfo: %#v",
			s.id, configuration.Layers, di)
	}

	configuration.LayerFolderPath = filepath.Join(di.HomeDir, s.id)
	if err := ioutil.WriteFile(filepath.Join(s.stateDir, layerFile), []byte(configuration.LayerFolderPath), 0644); err != nil {
		log.L.WithError(err).Warnf("failed to save active layer %s", configuration.LayerFolderPath)
	}

	err := hcsshim.ActivateLayer(di, s.id)
	if err != nil {
		removeLayer(context.TODO(), configuration.LayerFolderPath)
		return nil, errors.Wrapf(err, "failed to active layer %s", configuration.LayerFolderPath)
	}

	err = hcsshim.PrepareLayer(di, s.id, s.conf.Layers)
	if err != nil {
		removeLayer(context.TODO(), configuration.LayerFolderPath)
		return nil, errors.Wrapf(err, "failed to prepare layer %s", configuration.LayerFolderPath)
	}

	volumePath, err := hcsshim.GetLayerMountPath(di, s.id)
	if err != nil {
		if err := hcsshim.DestroyLayer(di, s.id); err != nil {
			log.L.Warnf("failed to DestroyLayer %s: %s", s.id, err)
		}
		return nil, errors.Wrapf(err, "failed to getmount path for layer %s: driverInfo: %#v", s.id, di)
	}
	configuration.VolumePath = volumePath

	f, err := os.OpenFile(fmt.Sprintf("%s-hcs.json", s.id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 066)
	if err != nil {
		fmt.Println("failed to create file:", err)
	} else {
		defer f.Close()
		enc := json.NewEncoder(f)
		enc.Encode(configuration)
	}

	return configuration, nil
}

// removeLayer delete the given layer, all associated containers must have
// been shutdown for this to succeed.
func removeLayer(ctx context.Context, path string) error {
	layerID := filepath.Base(path)
	parentPath := filepath.Dir(path)
	di := hcsshim.DriverInfo{
		Flavour: 1, // filter driver
		HomeDir: parentPath,
	}

	err := hcsshim.UnprepareLayer(di, layerID)
	if err != nil {
		log.G(ctx).WithError(err).Warnf("failed to unprepare layer %s for removal", path)
	}

	err = hcsshim.DeactivateLayer(di, layerID)
	if err != nil {
		log.G(ctx).WithError(err).Warnf("failed to deactivate layer %s for removal", path)
	}

	removePath := filepath.Join(parentPath, fmt.Sprintf("%s-removing", layerID))
	err = os.Rename(path, removePath)
	if err != nil {
		log.G(ctx).WithError(err).Warnf("failed to rename container layer %s for removal", path)
		removePath = path
	}
	if err := hcsshim.DestroyLayer(di, removePath); err != nil {
		log.G(ctx).WithError(err).Errorf("failed to remove container layer %s", removePath)
		return err
	}

	return nil
}

// ociSpecEnvToHCSEnv converts from the OCI Spec ENV format to the one
// expected by HCS.
func ociSpecEnvToHCSEnv(a []string) map[string]string {
	env := make(map[string]string)
	for _, s := range a {
		arr := strings.SplitN(s, "=", 2)
		if len(arr) == 2 {
			env[arr[0]] = arr[1]
		}
	}
	return env
}