execution: remove statedir type
Signed-off-by: Kenfe-Mickael Laventure <mickael.laventure@gmail.com>
This commit is contained in:
Kenfe-Mickael Laventure
parent
3f2d9d19bf
commit
bd6057c8e1
5 changed files with 308 additions and 273 deletions
|
|
@ -27,34 +27,37 @@ type newProcessOpts struct {
|
|||
runtimeArgs []string
|
||||
container *execution.Container
|
||||
exec bool
|
||||
stateDir string
|
||||
execution.StartProcessOpts
|
||||
}
|
||||
|
||||
func newProcess(ctx context.Context, o newProcessOpts) (*process, error) {
|
||||
procStateDir, err := o.container.StateDir().NewProcess(o.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func newProcess(ctx context.Context, o newProcessOpts) (p *process, err error) {
|
||||
p = &process{
|
||||
id: o.ID,
|
||||
stateDir: o.stateDir,
|
||||
exitChan: make(chan struct{}),
|
||||
ctx: ctx,
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
o.container.StateDir().DeleteProcess(o.ID)
|
||||
p.cleanup()
|
||||
p = nil
|
||||
}
|
||||
}()
|
||||
|
||||
exitPipe, controlPipe, err := getControlPipes(procStateDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if err = os.Mkdir(o.stateDir, 0700); err != nil {
|
||||
err = errors.Wrap(err, "failed to create process state dir")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
exitPipe.Close()
|
||||
controlPipe.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
cmd, err := newShim(o, procStateDir)
|
||||
p.exitPipe, p.controlPipe, err = getControlPipes(o.stateDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
|
||||
cmd, err := newShimProcess(o)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
|
|
@ -75,70 +78,52 @@ func newProcess(ctx context.Context, o newProcessOpts) (*process, error) {
|
|||
close(abortCh)
|
||||
}()
|
||||
|
||||
process := &process{
|
||||
root: procStateDir,
|
||||
id: o.ID,
|
||||
exitChan: make(chan struct{}),
|
||||
exitPipe: exitPipe,
|
||||
controlPipe: controlPipe,
|
||||
}
|
||||
|
||||
pid, stime, status, err := waitForPid(ctx, abortCh, procStateDir)
|
||||
p.pid, p.startTime, p.status, err = waitUntilReady(ctx, abortCh, o.stateDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
process.pid = int64(pid)
|
||||
process.status = status
|
||||
process.startTime = stime
|
||||
|
||||
return process, nil
|
||||
return
|
||||
}
|
||||
|
||||
func loadProcess(root, id string) (*process, error) {
|
||||
pid, err := runc.ReadPidFile(filepath.Join(root, pidFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stime, err := ioutil.ReadFile(filepath.Join(root, startTimeFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
path := filepath.Join(root, exitPipeFilename)
|
||||
exitPipe, err := os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func loadProcess(ctx context.Context, stateDir, id string) (p *process, err error) {
|
||||
p = &process{
|
||||
id: id,
|
||||
stateDir: stateDir,
|
||||
exitChan: make(chan struct{}),
|
||||
status: execution.Running,
|
||||
ctx: ctx,
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
exitPipe.Close()
|
||||
p.cleanup()
|
||||
p = nil
|
||||
}
|
||||
}()
|
||||
|
||||
path = filepath.Join(root, controlPipeFilename)
|
||||
controlPipe, err := os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0)
|
||||
p.pid, err = getPidFromFile(filepath.Join(stateDir, pidFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
err = errors.Wrap(err, "failed to read pid")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
controlPipe.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
p := &process{
|
||||
root: root,
|
||||
id: id,
|
||||
pid: int64(pid),
|
||||
exitChan: make(chan struct{}),
|
||||
exitPipe: exitPipe,
|
||||
controlPipe: controlPipe,
|
||||
startTime: string(stime),
|
||||
// TODO: status may need to be stored on disk to handle
|
||||
// Created state for init (i.e. a Start is needed to run the
|
||||
// container)
|
||||
status: execution.Running,
|
||||
p.startTime, err = getStartTimeFromFile(filepath.Join(stateDir, startTimeFilename))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
path := filepath.Join(stateDir, exitPipeFilename)
|
||||
p.exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "failed to open exit pipe")
|
||||
return
|
||||
}
|
||||
|
||||
path = filepath.Join(stateDir, controlPipeFilename)
|
||||
p.controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "failed to open control pipe")
|
||||
return
|
||||
}
|
||||
|
||||
markAsStopped := func(p *process) (*process, error) {
|
||||
|
|
@ -146,30 +131,32 @@ func loadProcess(root, id string) (*process, error) {
|
|||
return p, nil
|
||||
}
|
||||
|
||||
if err = syscall.Kill(pid, 0); err != nil {
|
||||
if err = syscall.Kill(int(p.pid), 0); err != nil {
|
||||
if err == syscall.ESRCH {
|
||||
return markAsStopped(p)
|
||||
}
|
||||
return nil, err
|
||||
err = errors.Wrapf(err, "failed to check if process is still alive")
|
||||
return
|
||||
}
|
||||
|
||||
cstime, err := starttime.GetProcessStartTime(pid)
|
||||
cstime, err := starttime.GetProcessStartTime(int(p.pid))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return markAsStopped(p)
|
||||
}
|
||||
return nil, err
|
||||
err = errors.Wrapf(err, "failed retrieve current process start time")
|
||||
return
|
||||
}
|
||||
|
||||
if p.startTime != cstime {
|
||||
return markAsStopped(p)
|
||||
}
|
||||
|
||||
return p, nil
|
||||
return
|
||||
}
|
||||
|
||||
type process struct {
|
||||
root string
|
||||
stateDir string
|
||||
id string
|
||||
pid int64
|
||||
exitChan chan struct{}
|
||||
|
|
@ -211,7 +198,7 @@ func (p *process) Wait() (uint32, error) {
|
|||
return uint32(128 + int(syscall.SIGKILL)), nil
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadFile(filepath.Join(p.root, exitStatusFilename))
|
||||
data, err := ioutil.ReadFile(filepath.Join(p.stateDir, exitStatusFilename))
|
||||
if err != nil {
|
||||
return execution.UnknownStatusCode, errors.Wrap(err, "failed to read process exit status")
|
||||
}
|
||||
|
|
@ -278,7 +265,19 @@ func (p *process) isAlive() bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root string) (pid int, stime string, status execution.Status, err error) {
|
||||
func (p *process) cleanup() {
|
||||
for _, f := range []*os.File{p.exitPipe, p.controlPipe} {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(p.stateDir); err != nil {
|
||||
log.G(p.ctx).Warnf("failed to remove process state dir: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitUntilReady(ctx context.Context, abortCh chan syscall.WaitStatus, root string) (pid int64, stime string, status execution.Status, err error) {
|
||||
status = execution.Unknown
|
||||
for {
|
||||
select {
|
||||
|
|
@ -293,7 +292,7 @@ func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root strin
|
|||
return
|
||||
default:
|
||||
}
|
||||
pid, err = runc.ReadPidFile(filepath.Join(root, pidFilename))
|
||||
pid, err = getPidFromFile(filepath.Join(root, pidFilename))
|
||||
if err == nil {
|
||||
break
|
||||
} else if !os.IsNotExist(err) {
|
||||
|
|
@ -301,7 +300,7 @@ func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root strin
|
|||
}
|
||||
}
|
||||
status = execution.Created
|
||||
stime, err = starttime.GetProcessStartTime(pid)
|
||||
stime, err = starttime.GetProcessStartTime(int(pid))
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
status = execution.Stopped
|
||||
|
|
@ -328,9 +327,9 @@ func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root strin
|
|||
return pid, stime, status, nil
|
||||
}
|
||||
|
||||
func newShim(o newProcessOpts, workDir string) (*exec.Cmd, error) {
|
||||
func newShimProcess(o newProcessOpts) (*exec.Cmd, error) {
|
||||
cmd := exec.Command(o.shimBinary, o.container.ID(), o.container.Bundle(), o.runtime)
|
||||
cmd.Dir = workDir
|
||||
cmd.Dir = o.stateDir
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Setpgid: true,
|
||||
}
|
||||
|
|
@ -348,12 +347,11 @@ func newShim(o newProcessOpts, workDir string) (*exec.Cmd, error) {
|
|||
RootGID: int(o.Spec.User.GID),
|
||||
}
|
||||
|
||||
f, err := os.Create(filepath.Join(workDir, "process.json"))
|
||||
f, err := os.Create(filepath.Join(o.stateDir, "process.json"))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create shim's process.json for container %s", o.container.ID())
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if err := json.NewEncoder(f).Encode(state); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create shim's processState for container %s", o.container.ID())
|
||||
}
|
||||
|
|
@ -368,19 +366,39 @@ func newShim(o newProcessOpts, workDir string) (*exec.Cmd, error) {
|
|||
func getControlPipes(root string) (exitPipe *os.File, controlPipe *os.File, err error) {
|
||||
path := filepath.Join(root, exitPipeFilename)
|
||||
if err = unix.Mkfifo(path, 0700); err != nil {
|
||||
return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim exit fifo")
|
||||
err = errors.Wrap(err, "failed to create shim exit fifo")
|
||||
return
|
||||
}
|
||||
if exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0); err != nil {
|
||||
return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim exit fifo")
|
||||
err = errors.Wrap(err, "failed to open shim exit fifo")
|
||||
return
|
||||
}
|
||||
|
||||
path = filepath.Join(root, controlPipeFilename)
|
||||
if err = unix.Mkfifo(path, 0700); err != nil {
|
||||
return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim control fifo")
|
||||
err = errors.Wrap(err, "failed to create shim control fifo")
|
||||
return
|
||||
}
|
||||
if controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0); err != nil {
|
||||
return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim control fifo")
|
||||
err = errors.Wrap(err, "failed to open shim control fifo")
|
||||
return
|
||||
}
|
||||
|
||||
return exitPipe, controlPipe, nil
|
||||
return
|
||||
}
|
||||
|
||||
func getPidFromFile(path string) (int64, error) {
|
||||
pid, err := runc.ReadPidFile(path)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return int64(pid), nil
|
||||
}
|
||||
|
||||
func getStartTimeFromFile(path string) (string, error) {
|
||||
stime, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to read start time")
|
||||
}
|
||||
return string(stime), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package shim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
|
|
@ -24,7 +25,6 @@ const (
|
|||
startTimeFilename = "starttime"
|
||||
exitPipeFilename = "exit"
|
||||
controlPipeFilename = "control"
|
||||
initProcessID = "init"
|
||||
exitStatusFilename = "exitStatus"
|
||||
)
|
||||
|
||||
|
|
@ -93,21 +93,17 @@ func (s *ShimRuntime) Create(ctx context.Context, id string, o execution.CreateO
|
|||
return nil, execution.ErrContainerExists
|
||||
}
|
||||
|
||||
container, err := execution.NewContainer(s.root, id, o.Bundle)
|
||||
containerCtx := log.WithModule(log.WithModule(ctx, "container"), id)
|
||||
container, err := execution.NewContainer(containerCtx, filepath.Join(s.root, id), id, o.Bundle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
container.StateDir().Delete()
|
||||
container.Cleanup()
|
||||
}
|
||||
}()
|
||||
|
||||
err = ioutil.WriteFile(filepath.Join(string(container.StateDir()), "bundle"), []byte(o.Bundle), 0600)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to save bundle path to disk")
|
||||
}
|
||||
|
||||
// extract Process spec from bundle's config.json
|
||||
var spec specs.Spec
|
||||
f, err := os.Open(filepath.Join(o.Bundle, "config.json"))
|
||||
|
|
@ -125,8 +121,9 @@ func (s *ShimRuntime) Create(ctx context.Context, id string, o execution.CreateO
|
|||
runtimeArgs: s.runtimeArgs,
|
||||
container: container,
|
||||
exec: false,
|
||||
stateDir: container.ProcessStateDir(execution.InitProcessID),
|
||||
StartProcessOpts: execution.StartProcessOpts{
|
||||
ID: initProcessID,
|
||||
ID: execution.InitProcessID,
|
||||
Spec: spec.Process,
|
||||
Console: o.Console,
|
||||
Stdin: o.Stdin,
|
||||
|
|
@ -135,14 +132,14 @@ func (s *ShimRuntime) Create(ctx context.Context, id string, o execution.CreateO
|
|||
},
|
||||
}
|
||||
|
||||
process, err := newProcess(ctx, processOpts)
|
||||
processCtx := log.WithModule(log.WithModule(containerCtx, "process"), execution.InitProcessID)
|
||||
process, err := newProcess(processCtx, processOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
process.ctx = log.WithModule(log.WithModule(s.ctx, "container"), id)
|
||||
|
||||
s.monitorProcess(process)
|
||||
container.AddProcess(process, true)
|
||||
container.AddProcess(process)
|
||||
|
||||
s.addContainer(container)
|
||||
|
||||
|
|
@ -194,7 +191,7 @@ func (s *ShimRuntime) Delete(ctx context.Context, c *execution.Container) error
|
|||
return errors.Errorf("cannot delete a container in the '%s' state", c.Status())
|
||||
}
|
||||
|
||||
c.StateDir().Delete()
|
||||
c.Cleanup()
|
||||
s.removeContainer(c)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -232,7 +229,9 @@ func (s *ShimRuntime) StartProcess(ctx context.Context, c *execution.Container,
|
|||
exec: true,
|
||||
StartProcessOpts: o,
|
||||
}
|
||||
process, err := newProcess(ctx, processOpts)
|
||||
|
||||
processCtx := log.WithModule(log.WithModule(c.Context(), "process"), execution.InitProcessID)
|
||||
process, err := newProcess(processCtx, processOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -240,7 +239,7 @@ func (s *ShimRuntime) StartProcess(ctx context.Context, c *execution.Container,
|
|||
process.status = execution.Running
|
||||
s.monitorProcess(process)
|
||||
|
||||
c.AddProcess(process, false)
|
||||
c.AddProcess(process)
|
||||
return process, nil
|
||||
}
|
||||
|
||||
|
|
@ -250,11 +249,11 @@ func (s *ShimRuntime) SignalProcess(ctx context.Context, c *execution.Container,
|
|||
|
||||
process := c.GetProcess(id)
|
||||
if process == nil {
|
||||
return errors.Errorf("no such process %s", id)
|
||||
return errors.Errorf("container %s has no process named %s", c.ID(), id)
|
||||
}
|
||||
err := syscall.Kill(int(process.Pid()), sig.(syscall.Signal))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to send %v signal to process %v", sig, process.Pid())
|
||||
return errors.Wrapf(err, "failed to send %v signal to container %s process %v", sig, c.ID(), process.Pid())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -263,13 +262,14 @@ func (s *ShimRuntime) DeleteProcess(ctx context.Context, c *execution.Container,
|
|||
log.G(s.ctx).WithFields(logrus.Fields{"container": c, "process-id": id}).
|
||||
Debug("DeleteProcess()")
|
||||
|
||||
c.RemoveProcess(id)
|
||||
return c.StateDir().DeleteProcess(id)
|
||||
}
|
||||
if p := c.GetProcess(id); p != nil {
|
||||
p.(*process).cleanup()
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
return c.RemoveProcess(id)
|
||||
}
|
||||
|
||||
return errors.Errorf("container %s has no process named %s", c.ID(), id)
|
||||
}
|
||||
|
||||
func (s *ShimRuntime) monitor() {
|
||||
var events [128]syscall.EpollEvent
|
||||
|
|
@ -375,46 +375,56 @@ func (s *ShimRuntime) loadContainers() {
|
|||
continue
|
||||
}
|
||||
|
||||
stateDir, err := execution.LoadStateDir(s.root, c.Name())
|
||||
stateDir := filepath.Join(s.root, c.Name())
|
||||
containerCtx := log.WithModule(log.WithModule(s.ctx, "container"), c.Name())
|
||||
container, err := execution.LoadContainer(containerCtx, stateDir, c.Name())
|
||||
if err != nil {
|
||||
// We should never fail the above call unless someone
|
||||
// delete the directory while we're loading
|
||||
log.G(s.ctx).WithFields(logrus.Fields{"container": c.Name(), "statedir": s.root}).
|
||||
Warn("failed to load container statedir:", err)
|
||||
continue
|
||||
}
|
||||
bundle, err := ioutil.ReadFile(filepath.Join(string(stateDir), "bundle"))
|
||||
if err != nil {
|
||||
log.G(s.ctx).WithField("container", c.Name()).
|
||||
Warn("failed to load container bundle path:", err)
|
||||
log.G(s.ctx).WithField("container-id", c.Name()).Warn(err)
|
||||
continue
|
||||
}
|
||||
|
||||
container := execution.LoadContainer(stateDir, c.Name(), string(bundle), execution.Unknown)
|
||||
s.addContainer(container)
|
||||
|
||||
processDirs, err := stateDir.Processes()
|
||||
processDirs, err := container.ProcessesStateDir()
|
||||
if err != nil {
|
||||
log.G(s.ctx).WithField("container", c.Name()).
|
||||
Warn("failed to retrieve container processes:", err)
|
||||
log.G(s.ctx).WithField("container-id", c.Name()).Warn(err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, procStateRoot := range processDirs {
|
||||
id := filepath.Base(procStateRoot)
|
||||
proc, err := loadProcess(procStateRoot, id)
|
||||
for processID, processStateDir := range processDirs {
|
||||
processCtx := log.WithModule(log.WithModule(containerCtx, "process"), processID)
|
||||
var p *process
|
||||
p, err = loadProcess(processCtx, processStateDir, processID)
|
||||
if err != nil {
|
||||
log.G(s.ctx).WithFields(logrus.Fields{"container": c.Name(), "process": id}).
|
||||
Warn("failed to load process:", err)
|
||||
s.removeContainer(container)
|
||||
for _, p := range container.Processes() {
|
||||
s.unmonitorProcess(p.(*process))
|
||||
}
|
||||
log.G(s.ctx).WithFields(logrus.Fields{"container-id": c.Name(), "process": processID}).Warn(err)
|
||||
break
|
||||
}
|
||||
proc.ctx = log.WithModule(log.WithModule(s.ctx, "container"), container.ID())
|
||||
container.AddProcess(proc, proc.ID() == initProcessID)
|
||||
s.monitorProcess(proc)
|
||||
if processID == execution.InitProcessID && p.status == execution.Running {
|
||||
p.status = s.loadContainerStatus(container.ID())
|
||||
}
|
||||
container.AddProcess(p)
|
||||
}
|
||||
|
||||
// if successfull, add the container to our list
|
||||
if err == nil {
|
||||
for _, p := range container.Processes() {
|
||||
s.monitorProcess(p.(*process))
|
||||
}
|
||||
s.addContainer(container)
|
||||
log.G(s.ctx).Infof("restored container %s", container.ID())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ShimRuntime) loadContainerStatus(id string) execution.Status {
|
||||
cmd := exec.Command(s.runtime, append(s.runtimeArgs, "state", id)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return execution.Unknown
|
||||
}
|
||||
|
||||
var st struct{ Status string }
|
||||
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&st); err != nil {
|
||||
return execution.Unknown
|
||||
}
|
||||
|
||||
return execution.Status(st.Status)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue