2015-12-18 00:07:04 +00:00
|
|
|
package supervisor
|
2015-11-05 23:29:53 +00:00
|
|
|
|
|
|
|
import (
|
2016-02-12 18:17:59 +00:00
|
|
|
"encoding/json"
|
|
|
|
"io"
|
2016-01-06 21:32:46 +00:00
|
|
|
"io/ioutil"
|
2015-11-05 23:29:53 +00:00
|
|
|
"os"
|
2016-02-12 18:17:59 +00:00
|
|
|
"path/filepath"
|
2015-12-04 21:31:17 +00:00
|
|
|
"sync"
|
2016-02-12 01:26:24 +00:00
|
|
|
"time"
|
2015-11-05 23:29:53 +00:00
|
|
|
|
|
|
|
"github.com/Sirupsen/logrus"
|
2015-12-01 19:56:08 +00:00
|
|
|
"github.com/docker/containerd/runtime"
|
2015-11-05 23:29:53 +00:00
|
|
|
)
|
|
|
|
|
2015-12-18 00:07:04 +00:00
|
|
|
const (
|
|
|
|
defaultBufferSize = 2048 // size of queue in eventloop
|
|
|
|
)
|
2015-12-14 22:43:00 +00:00
|
|
|
|
2015-12-18 00:07:04 +00:00
|
|
|
// New returns an initialized Process supervisor.
|
2016-04-19 00:28:13 +00:00
|
|
|
func New(stateDir string, runtimeName, shimName string, runtimeArgs []string, timeout time.Duration, retainCount int) (*Supervisor, error) {
|
2016-02-17 18:55:54 +00:00
|
|
|
startTasks := make(chan *startTask, 10)
|
2015-11-05 23:29:53 +00:00
|
|
|
if err := os.MkdirAll(stateDir, 0755); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
machine, err := CollectMachineInformation()
|
2015-11-05 23:49:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-11-05 23:29:53 +00:00
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
monitor, err := NewMonitor()
|
2015-12-03 19:49:56 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-11-05 23:49:13 +00:00
|
|
|
s := &Supervisor{
|
2016-02-11 22:36:32 +00:00
|
|
|
stateDir: stateDir,
|
|
|
|
containers: make(map[string]*containerInfo),
|
2016-02-17 18:55:54 +00:00
|
|
|
startTasks: startTasks,
|
2016-02-11 22:36:32 +00:00
|
|
|
machine: machine,
|
2016-02-12 01:26:24 +00:00
|
|
|
subscribers: make(map[chan Event]struct{}),
|
2016-02-17 18:55:54 +00:00
|
|
|
tasks: make(chan Task, defaultBufferSize),
|
2016-02-11 22:36:32 +00:00
|
|
|
monitor: monitor,
|
2016-02-25 20:59:34 +00:00
|
|
|
runtime: runtimeName,
|
2016-03-24 20:30:27 +00:00
|
|
|
runtimeArgs: runtimeArgs,
|
2016-04-06 06:42:47 +00:00
|
|
|
shim: shimName,
|
2016-03-28 20:30:37 +00:00
|
|
|
timeout: timeout,
|
2015-11-07 00:44:52 +00:00
|
|
|
}
|
2016-04-19 00:28:13 +00:00
|
|
|
if err := setupEventLog(s, retainCount); err != nil {
|
2016-02-12 18:17:59 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
go s.exitHandler()
|
2016-02-29 22:15:16 +00:00
|
|
|
go s.oomHandler()
|
2016-01-06 21:32:46 +00:00
|
|
|
if err := s.restore(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-11-05 23:29:53 +00:00
|
|
|
return s, nil
|
|
|
|
}
|
|
|
|
|
2015-12-11 19:27:33 +00:00
|
|
|
type containerInfo struct {
|
|
|
|
container runtime.Container
|
|
|
|
}
|
|
|
|
|
2016-04-19 00:28:13 +00:00
|
|
|
func setupEventLog(s *Supervisor, retainCount int) error {
|
2016-02-12 18:17:59 +00:00
|
|
|
if err := readEventLog(s); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
logrus.WithField("count", len(s.eventLog)).Debug("containerd: read past events")
|
|
|
|
events := s.Events(time.Time{})
|
2016-04-19 00:28:13 +00:00
|
|
|
return eventLogger(s, filepath.Join(s.stateDir, "events.log"), events, retainCount)
|
|
|
|
}
|
|
|
|
|
|
|
|
func eventLogger(s *Supervisor, path string, events chan Event, retainCount int) error {
|
|
|
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND|os.O_TRUNC, 0755)
|
2016-02-12 18:17:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
go func() {
|
2016-04-19 00:28:13 +00:00
|
|
|
var (
|
|
|
|
count = len(s.eventLog)
|
|
|
|
enc = json.NewEncoder(f)
|
|
|
|
)
|
2016-02-12 18:17:59 +00:00
|
|
|
for e := range events {
|
2016-04-19 00:28:13 +00:00
|
|
|
// if we have a specified retain count make sure the truncate the event
|
|
|
|
// log if it grows past the specified number of events to keep.
|
|
|
|
if retainCount > 0 {
|
|
|
|
if count > retainCount {
|
|
|
|
logrus.Debug("truncating event log")
|
|
|
|
// close the log file
|
|
|
|
if f != nil {
|
|
|
|
f.Close()
|
|
|
|
}
|
|
|
|
slice := retainCount - 1
|
|
|
|
l := len(s.eventLog)
|
|
|
|
if slice >= l {
|
|
|
|
slice = l
|
|
|
|
}
|
|
|
|
s.eventLock.Lock()
|
|
|
|
s.eventLog = s.eventLog[len(s.eventLog)-slice:]
|
|
|
|
s.eventLock.Unlock()
|
|
|
|
if f, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND|os.O_TRUNC, 0755); err != nil {
|
|
|
|
logrus.WithField("error", err).Error("containerd: open event to journal")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
enc = json.NewEncoder(f)
|
|
|
|
count = 0
|
|
|
|
for _, le := range s.eventLog {
|
|
|
|
if err := enc.Encode(le); err != nil {
|
|
|
|
logrus.WithField("error", err).Error("containerd: write event to journal")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
s.eventLock.Lock()
|
2016-02-12 18:17:59 +00:00
|
|
|
s.eventLog = append(s.eventLog, e)
|
2016-04-19 00:28:13 +00:00
|
|
|
s.eventLock.Unlock()
|
|
|
|
count++
|
2016-02-12 18:17:59 +00:00
|
|
|
if err := enc.Encode(e); err != nil {
|
|
|
|
logrus.WithField("error", err).Error("containerd: write event to journal")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func readEventLog(s *Supervisor) error {
|
|
|
|
f, err := os.Open(filepath.Join(s.stateDir, "events.log"))
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
dec := json.NewDecoder(f)
|
|
|
|
for {
|
|
|
|
var e Event
|
|
|
|
if err := dec.Decode(&e); err != nil {
|
|
|
|
if err == io.EOF {
|
2016-03-22 16:47:03 +00:00
|
|
|
break
|
2016-02-12 18:17:59 +00:00
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
s.eventLog = append(s.eventLog, e)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-06-03 22:00:49 +00:00
|
|
|
// Supervisor represents a container supervisor
|
2015-11-05 23:29:53 +00:00
|
|
|
type Supervisor struct {
|
|
|
|
// stateDir is the directory on the system to store container runtime state information.
|
2016-02-25 20:59:34 +00:00
|
|
|
stateDir string
|
|
|
|
// name of the OCI compatible runtime used to execute containers
|
2016-03-24 20:30:27 +00:00
|
|
|
runtime string
|
|
|
|
runtimeArgs []string
|
2016-04-06 06:42:47 +00:00
|
|
|
shim string
|
2016-03-24 20:30:27 +00:00
|
|
|
containers map[string]*containerInfo
|
|
|
|
startTasks chan *startTask
|
2015-12-10 22:11:00 +00:00
|
|
|
// we need a lock around the subscribers map only because additions and deletions from
|
|
|
|
// the map are via the API so we cannot really control the concurrency
|
2015-12-10 21:56:49 +00:00
|
|
|
subscriberLock sync.RWMutex
|
2016-02-12 01:26:24 +00:00
|
|
|
subscribers map[chan Event]struct{}
|
2015-12-04 21:31:17 +00:00
|
|
|
machine Machine
|
2016-02-17 18:55:54 +00:00
|
|
|
tasks chan Task
|
2016-01-06 21:32:46 +00:00
|
|
|
monitor *Monitor
|
2016-02-12 18:17:59 +00:00
|
|
|
eventLog []Event
|
2016-04-19 00:28:13 +00:00
|
|
|
eventLock sync.Mutex
|
2016-03-28 20:30:37 +00:00
|
|
|
timeout time.Duration
|
2015-11-05 23:29:53 +00:00
|
|
|
}
|
|
|
|
|
2016-02-17 18:55:54 +00:00
|
|
|
// Stop closes all startTasks and sends a SIGTERM to each container's pid1 then waits for they to
|
2015-12-10 22:11:00 +00:00
|
|
|
// terminate. After it has handled all the SIGCHILD events it will close the signals chan
|
|
|
|
// and exit. Stop is a non-blocking call and will return after the containers have been signaled
|
2016-01-06 21:32:46 +00:00
|
|
|
func (s *Supervisor) Stop() {
|
2016-02-17 18:55:54 +00:00
|
|
|
// Close the startTasks channel so that no new containers get started
|
|
|
|
close(s.startTasks)
|
2015-12-04 21:31:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Close closes any open files in the supervisor but expects that Stop has been
|
|
|
|
// callsed so that no more containers are started.
|
2015-11-10 22:24:34 +00:00
|
|
|
func (s *Supervisor) Close() error {
|
2015-12-07 22:24:40 +00:00
|
|
|
return nil
|
2015-11-10 22:24:34 +00:00
|
|
|
}
|
|
|
|
|
2016-06-03 22:00:49 +00:00
|
|
|
// Event represents a container event
|
2016-02-12 01:26:24 +00:00
|
|
|
type Event struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Timestamp time.Time `json:"timestamp"`
|
2016-02-17 18:55:54 +00:00
|
|
|
PID string `json:"pid,omitempty"`
|
2016-02-12 01:26:24 +00:00
|
|
|
Status int `json:"status,omitempty"`
|
|
|
|
}
|
|
|
|
|
2015-12-10 22:11:00 +00:00
|
|
|
// Events returns an event channel that external consumers can use to receive updates
|
|
|
|
// on container events
|
2016-02-12 18:17:59 +00:00
|
|
|
func (s *Supervisor) Events(from time.Time) chan Event {
|
2015-12-10 21:56:49 +00:00
|
|
|
s.subscriberLock.Lock()
|
|
|
|
defer s.subscriberLock.Unlock()
|
2016-02-12 01:26:24 +00:00
|
|
|
c := make(chan Event, defaultBufferSize)
|
2015-12-10 20:30:04 +00:00
|
|
|
EventSubscriberCounter.Inc(1)
|
|
|
|
s.subscribers[c] = struct{}{}
|
2016-02-12 18:17:59 +00:00
|
|
|
if !from.IsZero() {
|
|
|
|
// replay old event
|
2016-04-19 00:28:13 +00:00
|
|
|
s.eventLock.Lock()
|
|
|
|
past := s.eventLog[:]
|
|
|
|
s.eventLock.Unlock()
|
|
|
|
for _, e := range past {
|
2016-02-12 18:17:59 +00:00
|
|
|
if e.Timestamp.After(from) {
|
|
|
|
c <- e
|
|
|
|
}
|
|
|
|
}
|
2016-02-29 19:18:11 +00:00
|
|
|
// Notify the client that from now on it's live events
|
|
|
|
c <- Event{
|
2016-04-19 11:02:37 +00:00
|
|
|
Type: StateLive,
|
2016-02-29 19:18:11 +00:00
|
|
|
Timestamp: time.Now(),
|
|
|
|
}
|
2016-02-26 00:39:54 +00:00
|
|
|
}
|
2015-12-10 20:30:04 +00:00
|
|
|
return c
|
2015-12-01 23:49:24 +00:00
|
|
|
}
|
|
|
|
|
2015-12-10 22:11:00 +00:00
|
|
|
// Unsubscribe removes the provided channel from receiving any more events
|
2016-02-12 01:26:24 +00:00
|
|
|
func (s *Supervisor) Unsubscribe(sub chan Event) {
|
2015-12-10 21:56:49 +00:00
|
|
|
s.subscriberLock.Lock()
|
|
|
|
defer s.subscriberLock.Unlock()
|
2015-12-01 23:49:24 +00:00
|
|
|
delete(s.subscribers, sub)
|
2015-12-10 21:56:49 +00:00
|
|
|
close(sub)
|
2015-12-10 20:30:04 +00:00
|
|
|
EventSubscriberCounter.Dec(1)
|
2015-12-01 23:49:24 +00:00
|
|
|
}
|
|
|
|
|
2015-12-10 22:11:00 +00:00
|
|
|
// notifySubscribers will send the provided event to the external subscribers
|
|
|
|
// of the events channel
|
2016-02-12 01:26:24 +00:00
|
|
|
func (s *Supervisor) notifySubscribers(e Event) {
|
2015-12-10 21:56:49 +00:00
|
|
|
s.subscriberLock.RLock()
|
|
|
|
defer s.subscriberLock.RUnlock()
|
2015-12-01 23:49:24 +00:00
|
|
|
for sub := range s.subscribers {
|
2015-12-10 21:56:49 +00:00
|
|
|
// do a non-blocking send for the channel
|
|
|
|
select {
|
|
|
|
case sub <- e:
|
|
|
|
default:
|
2016-02-12 18:17:59 +00:00
|
|
|
logrus.WithField("event", e.Type).Warn("containerd: event not sent to subscriber")
|
2015-12-10 21:56:49 +00:00
|
|
|
}
|
2015-12-01 23:49:24 +00:00
|
|
|
}
|
2015-11-30 23:46:36 +00:00
|
|
|
}
|
|
|
|
|
2015-11-06 00:40:57 +00:00
|
|
|
// Start is a non-blocking call that runs the supervisor for monitoring contianer processes and
|
2015-11-05 23:29:53 +00:00
|
|
|
// executing new containers.
|
|
|
|
//
|
2015-12-10 22:11:00 +00:00
|
|
|
// This event loop is the only thing that is allowed to modify state of containers and processes
|
|
|
|
// therefore it is save to do operations in the handlers that modify state of the system or
|
|
|
|
// state of the Supervisor
|
2015-12-03 01:44:39 +00:00
|
|
|
func (s *Supervisor) Start() error {
|
2016-02-25 20:59:34 +00:00
|
|
|
logrus.WithFields(logrus.Fields{
|
2016-03-24 20:30:27 +00:00
|
|
|
"stateDir": s.stateDir,
|
|
|
|
"runtime": s.runtime,
|
|
|
|
"runtimeArgs": s.runtimeArgs,
|
|
|
|
"memory": s.machine.Memory,
|
|
|
|
"cpus": s.machine.Cpus,
|
2016-02-25 20:59:34 +00:00
|
|
|
}).Debug("containerd: supervisor running")
|
2016-02-17 18:55:54 +00:00
|
|
|
go func() {
|
|
|
|
for i := range s.tasks {
|
|
|
|
s.handleTask(i)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-12-03 19:49:56 +00:00
|
|
|
// Machine returns the machine information for which the
|
|
|
|
// supervisor is executing on.
|
|
|
|
func (s *Supervisor) Machine() Machine {
|
|
|
|
return s.machine
|
|
|
|
}
|
|
|
|
|
2016-02-12 01:26:24 +00:00
|
|
|
// SendTask sends the provided event the the supervisors main event loop
|
2016-02-17 18:55:54 +00:00
|
|
|
func (s *Supervisor) SendTask(evt Task) {
|
2016-02-12 01:26:24 +00:00
|
|
|
TasksCounter.Inc(1)
|
2016-02-17 18:55:54 +00:00
|
|
|
s.tasks <- evt
|
2015-11-05 23:29:53 +00:00
|
|
|
}
|
2015-12-11 01:07:21 +00:00
|
|
|
|
2016-01-06 21:32:46 +00:00
|
|
|
func (s *Supervisor) exitHandler() {
|
|
|
|
for p := range s.monitor.Exits() {
|
2016-04-27 19:00:29 +00:00
|
|
|
p.Wait()
|
2016-02-17 18:55:54 +00:00
|
|
|
e := &ExitTask{
|
|
|
|
Process: p,
|
|
|
|
}
|
2016-02-12 01:26:24 +00:00
|
|
|
s.SendTask(e)
|
2015-12-11 01:07:21 +00:00
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
}
|
|
|
|
|
2016-02-29 22:15:16 +00:00
|
|
|
func (s *Supervisor) oomHandler() {
|
|
|
|
for id := range s.monitor.OOMs() {
|
|
|
|
e := &OOMTask{
|
|
|
|
ID: id,
|
|
|
|
}
|
|
|
|
s.SendTask(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-06 21:32:46 +00:00
|
|
|
func (s *Supervisor) monitorProcess(p runtime.Process) error {
|
|
|
|
return s.monitor.Monitor(p)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Supervisor) restore() error {
|
|
|
|
dirs, err := ioutil.ReadDir(s.stateDir)
|
2015-12-11 19:27:33 +00:00
|
|
|
if err != nil {
|
2016-01-06 21:32:46 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
for _, d := range dirs {
|
|
|
|
if !d.IsDir() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
id := d.Name()
|
2016-07-05 11:43:56 +00:00
|
|
|
container, err := runtime.Load(s.stateDir, id, s.shim, s.timeout)
|
2016-01-06 21:32:46 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
processes, err := container.Processes()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-02-29 22:15:16 +00:00
|
|
|
|
2016-01-06 21:32:46 +00:00
|
|
|
ContainersCounter.Inc(1)
|
|
|
|
s.containers[id] = &containerInfo{
|
|
|
|
container: container,
|
|
|
|
}
|
2016-02-29 22:15:16 +00:00
|
|
|
if err := s.monitor.MonitorOOM(container); err != nil && err != runtime.ErrContainerExited {
|
|
|
|
logrus.WithField("error", err).Error("containerd: notify OOM events")
|
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
logrus.WithField("id", id).Debug("containerd: container restored")
|
2016-02-01 19:02:41 +00:00
|
|
|
var exitedProcesses []runtime.Process
|
2016-01-06 21:32:46 +00:00
|
|
|
for _, p := range processes {
|
2016-03-14 20:57:28 +00:00
|
|
|
if p.State() == runtime.Running {
|
2016-02-01 19:02:41 +00:00
|
|
|
if err := s.monitorProcess(p); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-03-14 20:57:28 +00:00
|
|
|
} else {
|
|
|
|
exitedProcesses = append(exitedProcesses, p)
|
2016-02-01 19:02:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(exitedProcesses) > 0 {
|
|
|
|
// sort processes so that init is fired last because that is how the kernel sends the
|
|
|
|
// exit events
|
2016-02-25 20:59:34 +00:00
|
|
|
sortProcesses(exitedProcesses)
|
2016-02-01 19:02:41 +00:00
|
|
|
for _, p := range exitedProcesses {
|
2016-02-17 18:55:54 +00:00
|
|
|
e := &ExitTask{
|
|
|
|
Process: p,
|
|
|
|
}
|
2016-02-12 01:26:24 +00:00
|
|
|
s.SendTask(e)
|
2016-01-06 21:32:46 +00:00
|
|
|
}
|
|
|
|
}
|
2015-12-11 01:07:21 +00:00
|
|
|
}
|
2016-01-06 21:32:46 +00:00
|
|
|
return nil
|
2015-12-11 01:07:21 +00:00
|
|
|
}
|
2016-04-26 20:29:35 +00:00
|
|
|
|
|
|
|
func (s *Supervisor) handleTask(i Task) {
|
|
|
|
var err error
|
|
|
|
switch t := i.(type) {
|
|
|
|
case *AddProcessTask:
|
|
|
|
err = s.addProcess(t)
|
|
|
|
case *CreateCheckpointTask:
|
|
|
|
err = s.createCheckpoint(t)
|
|
|
|
case *DeleteCheckpointTask:
|
|
|
|
err = s.deleteCheckpoint(t)
|
|
|
|
case *StartTask:
|
|
|
|
err = s.start(t)
|
|
|
|
case *DeleteTask:
|
|
|
|
err = s.delete(t)
|
|
|
|
case *ExitTask:
|
|
|
|
err = s.exit(t)
|
|
|
|
case *ExecExitTask:
|
|
|
|
err = s.execExit(t)
|
|
|
|
case *GetContainersTask:
|
|
|
|
err = s.getContainers(t)
|
|
|
|
case *SignalTask:
|
|
|
|
err = s.signal(t)
|
|
|
|
case *StatsTask:
|
|
|
|
err = s.stats(t)
|
|
|
|
case *UpdateTask:
|
|
|
|
err = s.updateContainer(t)
|
|
|
|
case *UpdateProcessTask:
|
|
|
|
err = s.updateProcess(t)
|
|
|
|
case *OOMTask:
|
|
|
|
err = s.oom(t)
|
|
|
|
default:
|
|
|
|
err = ErrUnknownTask
|
|
|
|
}
|
|
|
|
if err != errDeferredResponse {
|
|
|
|
i.ErrorCh() <- err
|
|
|
|
close(i.ErrorCh())
|
|
|
|
}
|
|
|
|
}
|