2017-01-12 18:32:09 +00:00
|
|
|
package shim
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"sync"
|
|
|
|
"syscall"
|
|
|
|
|
|
|
|
"github.com/docker/containerd/execution"
|
|
|
|
"github.com/docker/containerd/log"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"golang.org/x/sys/unix"
|
|
|
|
|
|
|
|
runc "github.com/crosbymichael/go-runc"
|
|
|
|
starttime "github.com/opencontainers/runc/libcontainer/system"
|
|
|
|
)
|
|
|
|
|
|
|
|
type newProcessOpts struct {
|
|
|
|
shimBinary string
|
|
|
|
runtime string
|
|
|
|
runtimeArgs []string
|
|
|
|
container *execution.Container
|
|
|
|
exec bool
|
2017-01-19 04:24:53 +00:00
|
|
|
stateDir string
|
2017-01-12 18:32:09 +00:00
|
|
|
execution.StartProcessOpts
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
func newProcess(ctx context.Context, o newProcessOpts) (p *process, err error) {
|
|
|
|
p = &process{
|
|
|
|
id: o.ID,
|
|
|
|
stateDir: o.stateDir,
|
|
|
|
exitChan: make(chan struct{}),
|
|
|
|
ctx: ctx,
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
2017-01-18 09:08:36 +00:00
|
|
|
defer func() {
|
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
p.cleanup()
|
|
|
|
p = nil
|
2017-01-18 09:08:36 +00:00
|
|
|
}
|
|
|
|
}()
|
2017-01-12 18:32:09 +00:00
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
if err = os.Mkdir(o.stateDir, 0700); err != nil {
|
|
|
|
err = errors.Wrap(err, "failed to create process state dir")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
p.exitPipe, p.controlPipe, err = getControlPipes(o.stateDir)
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
cmd, err := newShimProcess(o)
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
defer func() {
|
|
|
|
if err != nil {
|
|
|
|
cmd.Process.Kill()
|
|
|
|
cmd.Wait()
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
abortCh := make(chan syscall.WaitStatus, 1)
|
|
|
|
go func() {
|
|
|
|
var shimStatus syscall.WaitStatus
|
|
|
|
if err := cmd.Wait(); err != nil {
|
|
|
|
shimStatus = execution.UnknownStatusCode
|
|
|
|
} else {
|
|
|
|
shimStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
|
|
|
|
}
|
|
|
|
abortCh <- shimStatus
|
|
|
|
close(abortCh)
|
|
|
|
}()
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
p.pid, p.startTime, p.status, err = waitUntilReady(ctx, abortCh, o.stateDir)
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
func loadProcess(ctx context.Context, stateDir, id string) (p *process, err error) {
|
|
|
|
p = &process{
|
|
|
|
id: id,
|
|
|
|
stateDir: stateDir,
|
|
|
|
exitChan: make(chan struct{}),
|
|
|
|
status: execution.Running,
|
|
|
|
ctx: ctx,
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
2017-01-19 04:24:53 +00:00
|
|
|
defer func() {
|
|
|
|
if err != nil {
|
|
|
|
p.cleanup()
|
|
|
|
p = nil
|
|
|
|
}
|
|
|
|
}()
|
2017-01-12 18:32:09 +00:00
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
p.pid, err = getPidFromFile(filepath.Join(stateDir, pidFilename))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrap(err, "failed to read pid")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
p.startTime, err = getStartTimeFromFile(filepath.Join(stateDir, startTimeFilename))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
path := filepath.Join(stateDir, exitPipeFilename)
|
|
|
|
p.exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrapf(err, "failed to open exit pipe")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
path = filepath.Join(stateDir, controlPipeFilename)
|
|
|
|
p.controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0)
|
|
|
|
if err != nil {
|
|
|
|
err = errors.Wrapf(err, "failed to open control pipe")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
markAsStopped := func(p *process) (*process, error) {
|
|
|
|
p.setStatus(execution.Stopped)
|
|
|
|
return p, nil
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
if err = syscall.Kill(int(p.pid), 0); err != nil {
|
2017-01-12 18:32:09 +00:00
|
|
|
if err == syscall.ESRCH {
|
|
|
|
return markAsStopped(p)
|
|
|
|
}
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrapf(err, "failed to check if process is still alive")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
cstime, err := starttime.GetProcessStartTime(int(p.pid))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return markAsStopped(p)
|
|
|
|
}
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrapf(err, "failed retrieve current process start time")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if p.startTime != cstime {
|
|
|
|
return markAsStopped(p)
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type process struct {
|
2017-01-19 04:24:53 +00:00
|
|
|
stateDir string
|
2017-01-12 18:32:09 +00:00
|
|
|
id string
|
|
|
|
pid int64
|
|
|
|
exitChan chan struct{}
|
|
|
|
exitPipe *os.File
|
|
|
|
controlPipe *os.File
|
|
|
|
startTime string
|
|
|
|
status execution.Status
|
|
|
|
ctx context.Context
|
|
|
|
mu sync.Mutex
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) ID() string {
|
|
|
|
return p.id
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) Pid() int64 {
|
|
|
|
return p.pid
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) Wait() (uint32, error) {
|
|
|
|
<-p.exitChan
|
|
|
|
|
|
|
|
log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
|
|
|
|
Debugf("wait is over")
|
|
|
|
|
|
|
|
// Cleanup those fds
|
|
|
|
p.exitPipe.Close()
|
|
|
|
p.controlPipe.Close()
|
|
|
|
|
|
|
|
// If the container process is still alive, it means the shim crashed
|
|
|
|
// and the child process had updated it PDEATHSIG to something
|
|
|
|
// else than SIGKILL. Or that epollCtl failed
|
|
|
|
if p.isAlive() {
|
|
|
|
err := syscall.Kill(int(p.pid), syscall.SIGKILL)
|
|
|
|
if err != nil {
|
|
|
|
return execution.UnknownStatusCode, errors.Wrap(err, "failed to kill process")
|
|
|
|
}
|
|
|
|
|
|
|
|
return uint32(128 + int(syscall.SIGKILL)), nil
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
data, err := ioutil.ReadFile(filepath.Join(p.stateDir, exitStatusFilename))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
|
|
|
return execution.UnknownStatusCode, errors.Wrap(err, "failed to read process exit status")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(data) == 0 {
|
|
|
|
return execution.UnknownStatusCode, errors.New(execution.ErrProcessNotExited.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
status, err := strconv.Atoi(string(data))
|
|
|
|
if err != nil {
|
|
|
|
return execution.UnknownStatusCode, errors.Wrapf(err, "failed to parse exit status")
|
|
|
|
}
|
|
|
|
|
|
|
|
p.setStatus(execution.Stopped)
|
|
|
|
return uint32(status), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) Signal(sig os.Signal) error {
|
|
|
|
err := syscall.Kill(int(p.pid), sig.(syscall.Signal))
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "failed to signal process")
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) Status() execution.Status {
|
|
|
|
p.mu.Lock()
|
|
|
|
s := p.status
|
|
|
|
p.mu.Unlock()
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) setStatus(s execution.Status) {
|
|
|
|
p.mu.Lock()
|
|
|
|
p.status = s
|
|
|
|
p.mu.Unlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *process) isAlive() bool {
|
|
|
|
if err := syscall.Kill(int(p.pid), 0); err != nil {
|
|
|
|
if err == syscall.ESRCH {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
|
|
|
|
Warnf("kill(0) failed: %v", err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2017-01-19 17:18:18 +00:00
|
|
|
// check that we have the same starttime
|
2017-01-12 18:32:09 +00:00
|
|
|
stime, err := starttime.GetProcessStartTime(int(p.pid))
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
|
|
|
|
Warnf("failed to get process start time: %v", err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.startTime != stime {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
func (p *process) cleanup() {
|
|
|
|
for _, f := range []*os.File{p.exitPipe, p.controlPipe} {
|
|
|
|
if f != nil {
|
|
|
|
f.Close()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.RemoveAll(p.stateDir); err != nil {
|
|
|
|
log.G(p.ctx).Warnf("failed to remove process state dir: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func waitUntilReady(ctx context.Context, abortCh chan syscall.WaitStatus, root string) (pid int64, stime string, status execution.Status, err error) {
|
2017-01-12 18:32:09 +00:00
|
|
|
status = execution.Unknown
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case wait := <-abortCh:
|
|
|
|
if wait.Signaled() {
|
2017-01-19 17:18:18 +00:00
|
|
|
err = errors.Errorf("shim died prematurely: %v", wait.Signal())
|
2017-01-12 18:32:09 +00:00
|
|
|
return
|
|
|
|
}
|
2017-01-19 17:18:18 +00:00
|
|
|
err = errors.Errorf("shim exited prematurely with exit code %v", wait.ExitStatus())
|
2017-01-12 18:32:09 +00:00
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
2017-01-19 04:24:53 +00:00
|
|
|
pid, err = getPidFromFile(filepath.Join(root, pidFilename))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err == nil {
|
|
|
|
break
|
|
|
|
} else if !os.IsNotExist(err) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
status = execution.Created
|
2017-01-19 04:24:53 +00:00
|
|
|
stime, err = starttime.GetProcessStartTime(int(pid))
|
2017-01-12 18:32:09 +00:00
|
|
|
switch {
|
|
|
|
case os.IsNotExist(err):
|
|
|
|
status = execution.Stopped
|
|
|
|
case err != nil:
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
var b []byte
|
|
|
|
path := filepath.Join(root, startTimeFilename)
|
|
|
|
b, err = ioutil.ReadFile(path)
|
|
|
|
switch {
|
|
|
|
case os.IsNotExist(err):
|
|
|
|
err = ioutil.WriteFile(path, []byte(stime), 0600)
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
case err != nil:
|
|
|
|
err = errors.Wrapf(err, "failed to get start time for pid %d", pid)
|
|
|
|
return
|
|
|
|
case string(b) != stime:
|
|
|
|
status = execution.Stopped
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return pid, stime, status, nil
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
func newShimProcess(o newProcessOpts) (*exec.Cmd, error) {
|
2017-01-12 18:32:09 +00:00
|
|
|
cmd := exec.Command(o.shimBinary, o.container.ID(), o.container.Bundle(), o.runtime)
|
2017-01-19 04:24:53 +00:00
|
|
|
cmd.Dir = o.stateDir
|
2017-01-12 18:32:09 +00:00
|
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{
|
|
|
|
Setpgid: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
state := processState{
|
|
|
|
Process: o.Spec,
|
|
|
|
Exec: o.exec,
|
|
|
|
Stdin: o.Stdin,
|
|
|
|
Stdout: o.Stdout,
|
|
|
|
Stderr: o.Stderr,
|
|
|
|
RuntimeArgs: o.runtimeArgs,
|
|
|
|
NoPivotRoot: false,
|
|
|
|
CheckpointPath: "",
|
|
|
|
RootUID: int(o.Spec.User.UID),
|
|
|
|
RootGID: int(o.Spec.User.GID),
|
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
f, err := os.Create(filepath.Join(o.stateDir, "process.json"))
|
2017-01-12 18:32:09 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "failed to create shim's process.json for container %s", o.container.ID())
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
if err := json.NewEncoder(f).Encode(state); err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "failed to create shim's processState for container %s", o.container.ID())
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "failed to start shim for container %s", o.container.ID())
|
|
|
|
}
|
|
|
|
|
|
|
|
return cmd, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getControlPipes(root string) (exitPipe *os.File, controlPipe *os.File, err error) {
|
|
|
|
path := filepath.Join(root, exitPipeFilename)
|
|
|
|
if err = unix.Mkfifo(path, 0700); err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrap(err, "failed to create shim exit fifo")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
if exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0); err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrap(err, "failed to open shim exit fifo")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
path = filepath.Join(root, controlPipeFilename)
|
|
|
|
if err = unix.Mkfifo(path, 0700); err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrap(err, "failed to create shim control fifo")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
if controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0); err != nil {
|
2017-01-19 04:24:53 +00:00
|
|
|
err = errors.Wrap(err, "failed to open shim control fifo")
|
|
|
|
return
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 04:24:53 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func getPidFromFile(path string) (int64, error) {
|
|
|
|
pid, err := runc.ReadPidFile(path)
|
|
|
|
if err != nil {
|
|
|
|
return -1, err
|
|
|
|
}
|
|
|
|
return int64(pid), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getStartTimeFromFile(path string) (string, error) {
|
|
|
|
stime, err := ioutil.ReadFile(path)
|
|
|
|
if err != nil {
|
|
|
|
return "", errors.Wrapf(err, "failed to read start time")
|
|
|
|
}
|
|
|
|
return string(stime), nil
|
2017-01-12 18:32:09 +00:00
|
|
|
}
|