121 lines
2.9 KiB
Go
121 lines
2.9 KiB
Go
|
/*
|
||
|
|
Copyright 2015 The Kubernetes Authors.
|
||
|
|
|
||
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
you may not use this file except in compliance with the License.
|
||
|
|
You may obtain a copy of the License at
|
||
|
|
|
||
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
|
||
|
|
Unless required by applicable law or agreed to in writing, software
|
||
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
See the License for the specific language governing permissions and
|
||
|
|
limitations under the License.
|
||
|
|
*/
|
||
|
|
|
||
|
|
package threading
|
||
|
|
|
||
|
|
import (
|
||
|
|
"os"
|
||
|
|
"sync"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/golang/glog"
|
||
|
|
)
|
||
|
|
|
||
|
|
type rwMutexToLockableAdapter struct {
|
||
|
|
rw *sync.RWMutex
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *rwMutexToLockableAdapter) Lock() {
|
||
|
|
r.rw.RLock()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *rwMutexToLockableAdapter) Unlock() {
|
||
|
|
r.rw.RUnlock()
|
||
|
|
}
|
||
|
|
|
||
|
|
type deadlockDetector struct {
|
||
|
|
name string
|
||
|
|
lock sync.Locker
|
||
|
|
maxLockPeriod time.Duration
|
||
|
|
exiter exiter
|
||
|
|
exitChannelFn func() <-chan time.Time
|
||
|
|
// Really only useful for testing
|
||
|
|
stopChannel <-chan bool
|
||
|
|
}
|
||
|
|
|
||
|
|
// DeadlockWatchdogReadLock creates a watchdog on read/write mutex. If the mutex can not be acquired
|
||
|
|
// for read access within 'maxLockPeriod', the program exits via glog.Exitf() or os.Exit() if that fails
|
||
|
|
// 'name' is a semantic name that is useful for the user and is printed on exit.
|
||
|
|
func DeadlockWatchdogReadLock(lock *sync.RWMutex, name string, maxLockPeriod time.Duration) {
|
||
|
|
DeadlockWatchdog(&rwMutexToLockableAdapter{lock}, name, maxLockPeriod)
|
||
|
|
}
|
||
|
|
|
||
|
|
func DeadlockWatchdog(lock sync.Locker, name string, maxLockPeriod time.Duration) {
|
||
|
|
if maxLockPeriod <= 0 {
|
||
|
|
panic("maxLockPeriod is <= 0, that can't be what you wanted")
|
||
|
|
}
|
||
|
|
detector := &deadlockDetector{
|
||
|
|
lock: lock,
|
||
|
|
name: name,
|
||
|
|
maxLockPeriod: maxLockPeriod,
|
||
|
|
exitChannelFn: func() <-chan time.Time { return time.After(maxLockPeriod) },
|
||
|
|
stopChannel: make(chan bool),
|
||
|
|
}
|
||
|
|
go detector.run()
|
||
|
|
}
|
||
|
|
|
||
|
|
// Useful for injecting tests
|
||
|
|
type exiter interface {
|
||
|
|
Exitf(format string, args ...interface{})
|
||
|
|
}
|
||
|
|
|
||
|
|
type realExiter struct{}
|
||
|
|
|
||
|
|
func (realExiter) Exitf(format string, args ...interface{}) {
|
||
|
|
func() {
|
||
|
|
defer func() {
|
||
|
|
// Let's just be extra sure we die, even if Exitf panics
|
||
|
|
if r := recover(); r != nil {
|
||
|
|
glog.Errorf(format, args...)
|
||
|
|
os.Exit(2)
|
||
|
|
}
|
||
|
|
}()
|
||
|
|
glog.Exitf(format, args...)
|
||
|
|
}()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *deadlockDetector) run() {
|
||
|
|
for {
|
||
|
|
if !d.runOnce() {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
time.Sleep(d.maxLockPeriod / 2)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *deadlockDetector) runOnce() bool {
|
||
|
|
ch := make(chan bool, 1)
|
||
|
|
go func() {
|
||
|
|
d.lock.Lock()
|
||
|
|
d.lock.Unlock()
|
||
|
|
|
||
|
|
ch <- true
|
||
|
|
}()
|
||
|
|
exitCh := d.exitChannelFn()
|
||
|
|
select {
|
||
|
|
case <-exitCh:
|
||
|
|
d.exiter.Exitf("Deadlock on %s, exiting", d.name)
|
||
|
|
// return is here for when we use a fake exiter in testing
|
||
|
|
return false
|
||
|
|
case <-ch:
|
||
|
|
glog.V(6).Infof("%s is not deadlocked", d.name)
|
||
|
|
case <-d.stopChannel:
|
||
|
|
glog.V(4).Infof("Stopping deadlock detector for %s", d.name)
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|