2017-02-01 00:45:59 +00:00
/ *
Copyright 2014 The Kubernetes Authors .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package node
import (
"errors"
"fmt"
"net"
"sync"
"time"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2017-02-03 13:41:32 +00:00
"k8s.io/apimachinery/pkg/fields"
2017-02-01 00:45:59 +00:00
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
2017-02-03 13:41:32 +00:00
"k8s.io/client-go/util/flowcontrol"
2017-02-01 00:45:59 +00:00
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/cache"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
2017-02-03 13:41:32 +00:00
"k8s.io/kubernetes/pkg/client/legacylisters"
2017-02-01 00:45:59 +00:00
"k8s.io/kubernetes/pkg/client/record"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/controller/informers"
"k8s.io/kubernetes/pkg/util/metrics"
utilnode "k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/util/system"
utilversion "k8s.io/kubernetes/pkg/util/version"
)
func init ( ) {
// Register prometheus metrics
Register ( )
}
var (
ErrCloudInstance = errors . New ( "cloud provider doesn't support instances." )
gracefulDeletionVersion = utilversion . MustParseSemantic ( "v1.1.0" )
// The minimum kubelet version for which the nodecontroller
// can safely flip pod.Status to NotReady.
podStatusReconciliationVersion = utilversion . MustParseSemantic ( "v1.2.0" )
)
const (
// nodeStatusUpdateRetry controls the number of retries of writing NodeStatus update.
nodeStatusUpdateRetry = 5
// controls how often NodeController will try to evict Pods from non-responsive Nodes.
nodeEvictionPeriod = 100 * time . Millisecond
// Burst value for all eviction rate limiters
evictionRateLimiterBurst = 1
// The amount of time the nodecontroller polls on the list nodes endpoint.
apiserverStartupGracePeriod = 10 * time . Minute
// The amount of time the nodecontroller should sleep between retrying NodeStatus updates
retrySleepTime = 20 * time . Millisecond
)
type zoneState string
const (
stateInitial = zoneState ( "Initial" )
stateNormal = zoneState ( "Normal" )
stateFullDisruption = zoneState ( "FullDisruption" )
statePartialDisruption = zoneState ( "PartialDisruption" )
)
type nodeStatusData struct {
probeTimestamp metav1 . Time
readyTransitionTimestamp metav1 . Time
status v1 . NodeStatus
}
type NodeController struct {
allocateNodeCIDRs bool
cloud cloudprovider . Interface
clusterCIDR * net . IPNet
serviceCIDR * net . IPNet
knownNodeSet map [ string ] * v1 . Node
kubeClient clientset . Interface
// Method for easy mocking in unittest.
lookupIP func ( host string ) ( [ ] net . IP , error )
// Value used if sync_nodes_status=False. NodeController will not proactively
// sync node status in this case, but will monitor node status updated from kubelet. If
// it doesn't receive update for this amount of time, it will start posting "NodeReady==
// ConditionUnknown". The amount of time before which NodeController start evicting pods
// is controlled via flag 'pod-eviction-timeout'.
// Note: be cautious when changing the constant, it must work with nodeStatusUpdateFrequency
// in kubelet. There are several constraints:
// 1. nodeMonitorGracePeriod must be N times more than nodeStatusUpdateFrequency, where
// N means number of retries allowed for kubelet to post node status. It is pointless
// to make nodeMonitorGracePeriod be less than nodeStatusUpdateFrequency, since there
// will only be fresh values from Kubelet at an interval of nodeStatusUpdateFrequency.
// The constant must be less than podEvictionTimeout.
// 2. nodeMonitorGracePeriod can't be too large for user experience - larger value takes
// longer for user to see up-to-date node status.
nodeMonitorGracePeriod time . Duration
// Value controlling NodeController monitoring period, i.e. how often does NodeController
// check node status posted from kubelet. This value should be lower than nodeMonitorGracePeriod.
// TODO: Change node status monitor to watch based.
nodeMonitorPeriod time . Duration
// Value used if sync_nodes_status=False, only for node startup. When node
// is just created, e.g. cluster bootstrap or node creation, we give a longer grace period.
nodeStartupGracePeriod time . Duration
// per Node map storing last observed Status together with a local time when it was observed.
// This timestamp is to be used instead of LastProbeTime stored in Condition. We do this
// to aviod the problem with time skew across the cluster.
nodeStatusMap map [ string ] nodeStatusData
now func ( ) metav1 . Time
// Lock to access evictor workers
evictorLock sync . Mutex
// workers that evicts pods from unresponsive nodes.
zonePodEvictor map [ string ] * RateLimitedTimedQueue
podEvictionTimeout time . Duration
// The maximum duration before a pod evicted from a node can be forcefully terminated.
maximumGracePeriod time . Duration
recorder record . EventRecorder
podInformer informers . PodInformer
nodeInformer informers . NodeInformer
daemonSetInformer informers . DaemonSetInformer
2017-02-03 13:41:32 +00:00
podStore listers . StoreToPodLister
nodeStore listers . StoreToNodeLister
daemonSetStore listers . StoreToDaemonSetLister
2017-02-01 00:45:59 +00:00
// allocate/recycle CIDRs for node if allocateNodeCIDRs == true
cidrAllocator CIDRAllocator
forcefullyDeletePod func ( * v1 . Pod ) error
nodeExistsInCloudProvider func ( types . NodeName ) ( bool , error )
computeZoneStateFunc func ( nodeConditions [ ] * v1 . NodeCondition ) ( int , zoneState )
enterPartialDisruptionFunc func ( nodeNum int ) float32
enterFullDisruptionFunc func ( nodeNum int ) float32
zoneStates map [ string ] zoneState
evictionLimiterQPS float32
secondaryEvictionLimiterQPS float32
largeClusterThreshold int32
unhealthyZoneThreshold float32
// internalPodInformer is used to hold a personal informer. If we're using
// a normal shared informer, then the informer will be started for us. If
// we have a personal informer, we must start it ourselves. If you start
// the controller using NewDaemonSetsController(passing SharedInformer), this
// will be null
internalPodInformer cache . SharedIndexInformer
}
// NewNodeController returns a new node controller to sync instances from cloudprovider.
// This method returns an error if it is unable to initialize the CIDR bitmap with
// podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes
// currently, this should be handled as a fatal error.
func NewNodeController (
podInformer informers . PodInformer ,
nodeInformer informers . NodeInformer ,
daemonSetInformer informers . DaemonSetInformer ,
cloud cloudprovider . Interface ,
kubeClient clientset . Interface ,
podEvictionTimeout time . Duration ,
evictionLimiterQPS float32 ,
secondaryEvictionLimiterQPS float32 ,
largeClusterThreshold int32 ,
unhealthyZoneThreshold float32 ,
nodeMonitorGracePeriod time . Duration ,
nodeStartupGracePeriod time . Duration ,
nodeMonitorPeriod time . Duration ,
clusterCIDR * net . IPNet ,
serviceCIDR * net . IPNet ,
nodeCIDRMaskSize int ,
allocateNodeCIDRs bool ) ( * NodeController , error ) {
eventBroadcaster := record . NewBroadcaster ( )
recorder := eventBroadcaster . NewRecorder ( v1 . EventSource { Component : "controllermanager" } )
eventBroadcaster . StartLogging ( glog . Infof )
if kubeClient != nil {
glog . V ( 0 ) . Infof ( "Sending events to api server." )
eventBroadcaster . StartRecordingToSink ( & v1core . EventSinkImpl { Interface : kubeClient . Core ( ) . Events ( "" ) } )
} else {
glog . V ( 0 ) . Infof ( "No api server defined - no events will be sent to API server." )
}
if kubeClient != nil && kubeClient . Core ( ) . RESTClient ( ) . GetRateLimiter ( ) != nil {
metrics . RegisterMetricAndTrackRateLimiterUsage ( "node_controller" , kubeClient . Core ( ) . RESTClient ( ) . GetRateLimiter ( ) )
}
if allocateNodeCIDRs {
if clusterCIDR == nil {
glog . Fatal ( "NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true." )
}
mask := clusterCIDR . Mask
if maskSize , _ := mask . Size ( ) ; maskSize > nodeCIDRMaskSize {
glog . Fatal ( "NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize." )
}
}
nc := & NodeController {
cloud : cloud ,
knownNodeSet : make ( map [ string ] * v1 . Node ) ,
kubeClient : kubeClient ,
recorder : recorder ,
podEvictionTimeout : podEvictionTimeout ,
maximumGracePeriod : 5 * time . Minute ,
zonePodEvictor : make ( map [ string ] * RateLimitedTimedQueue ) ,
nodeStatusMap : make ( map [ string ] nodeStatusData ) ,
nodeMonitorGracePeriod : nodeMonitorGracePeriod ,
nodeMonitorPeriod : nodeMonitorPeriod ,
nodeStartupGracePeriod : nodeStartupGracePeriod ,
lookupIP : net . LookupIP ,
now : metav1 . Now ,
clusterCIDR : clusterCIDR ,
serviceCIDR : serviceCIDR ,
allocateNodeCIDRs : allocateNodeCIDRs ,
forcefullyDeletePod : func ( p * v1 . Pod ) error { return forcefullyDeletePod ( kubeClient , p ) } ,
nodeExistsInCloudProvider : func ( nodeName types . NodeName ) ( bool , error ) { return nodeExistsInCloudProvider ( cloud , nodeName ) } ,
evictionLimiterQPS : evictionLimiterQPS ,
secondaryEvictionLimiterQPS : secondaryEvictionLimiterQPS ,
largeClusterThreshold : largeClusterThreshold ,
unhealthyZoneThreshold : unhealthyZoneThreshold ,
zoneStates : make ( map [ string ] zoneState ) ,
podInformer : podInformer ,
nodeInformer : nodeInformer ,
daemonSetInformer : daemonSetInformer ,
}
nc . enterPartialDisruptionFunc = nc . ReducedQPSFunc
nc . enterFullDisruptionFunc = nc . HealthyQPSFunc
nc . computeZoneStateFunc = nc . ComputeZoneState
podInformer . Informer ( ) . AddEventHandler ( cache . ResourceEventHandlerFuncs {
AddFunc : nc . maybeDeleteTerminatingPod ,
UpdateFunc : func ( _ , obj interface { } ) { nc . maybeDeleteTerminatingPod ( obj ) } ,
} )
nc . podStore = * podInformer . Lister ( )
nodeEventHandlerFuncs := cache . ResourceEventHandlerFuncs { }
if nc . allocateNodeCIDRs {
var nodeList * v1 . NodeList
var err error
// We must poll because apiserver might not be up. This error causes
// controller manager to restart.
if pollErr := wait . Poll ( 10 * time . Second , apiserverStartupGracePeriod , func ( ) ( bool , error ) {
2017-02-03 13:41:32 +00:00
nodeList , err = kubeClient . Core ( ) . Nodes ( ) . List ( metav1 . ListOptions {
2017-02-01 00:45:59 +00:00
FieldSelector : fields . Everything ( ) . String ( ) ,
LabelSelector : labels . Everything ( ) . String ( ) ,
} )
if err != nil {
glog . Errorf ( "Failed to list all nodes: %v" , err )
return false , nil
}
return true , nil
} ) ; pollErr != nil {
return nil , fmt . Errorf ( "Failed to list all nodes in %v, cannot proceed without updating CIDR map" , apiserverStartupGracePeriod )
}
nc . cidrAllocator , err = NewCIDRRangeAllocator ( kubeClient , clusterCIDR , serviceCIDR , nodeCIDRMaskSize , nodeList )
if err != nil {
return nil , err
}
nodeEventHandlerFuncs = cache . ResourceEventHandlerFuncs {
AddFunc : func ( originalObj interface { } ) {
obj , err := api . Scheme . DeepCopy ( originalObj )
if err != nil {
utilruntime . HandleError ( err )
return
}
node := obj . ( * v1 . Node )
if err := nc . cidrAllocator . AllocateOrOccupyCIDR ( node ) ; err != nil {
utilruntime . HandleError ( fmt . Errorf ( "Error allocating CIDR: %v" , err ) )
}
} ,
UpdateFunc : func ( _ , obj interface { } ) {
node := obj . ( * v1 . Node )
// If the PodCIDR is not empty we either:
// - already processed a Node that already had a CIDR after NC restarted
// (cidr is marked as used),
// - already processed a Node successfully and allocated a CIDR for it
// (cidr is marked as used),
// - already processed a Node but we did saw a "timeout" response and
// request eventually got through in this case we haven't released
// the allocated CIDR (cidr is still marked as used).
// There's a possible error here:
// - NC sees a new Node and assigns a CIDR X to it,
// - Update Node call fails with a timeout,
// - Node is updated by some other component, NC sees an update and
// assigns CIDR Y to the Node,
// - Both CIDR X and CIDR Y are marked as used in the local cache,
// even though Node sees only CIDR Y
// The problem here is that in in-memory cache we see CIDR X as marked,
// which prevents it from being assigned to any new node. The cluster
// state is correct.
// Restart of NC fixes the issue.
if node . Spec . PodCIDR == "" {
nodeCopy , err := api . Scheme . Copy ( node )
if err != nil {
utilruntime . HandleError ( err )
return
}
if err := nc . cidrAllocator . AllocateOrOccupyCIDR ( nodeCopy . ( * v1 . Node ) ) ; err != nil {
utilruntime . HandleError ( fmt . Errorf ( "Error allocating CIDR: %v" , err ) )
}
}
} ,
DeleteFunc : func ( originalObj interface { } ) {
obj , err := api . Scheme . DeepCopy ( originalObj )
if err != nil {
utilruntime . HandleError ( err )
return
}
node , isNode := obj . ( * v1 . Node )
// We can get DeletedFinalStateUnknown instead of *v1.Node here and we need to handle that correctly. #34692
if ! isNode {
deletedState , ok := obj . ( cache . DeletedFinalStateUnknown )
if ! ok {
glog . Errorf ( "Received unexpected object: %v" , obj )
return
}
node , ok = deletedState . Obj . ( * v1 . Node )
if ! ok {
glog . Errorf ( "DeletedFinalStateUnknown contained non-Node object: %v" , deletedState . Obj )
return
}
}
if err := nc . cidrAllocator . ReleaseCIDR ( node ) ; err != nil {
glog . Errorf ( "Error releasing CIDR: %v" , err )
}
} ,
}
}
nodeInformer . Informer ( ) . AddEventHandler ( nodeEventHandlerFuncs )
nc . nodeStore = * nodeInformer . Lister ( )
nc . daemonSetStore = * daemonSetInformer . Lister ( )
return nc , nil
}
// Run starts an asynchronous loop that monitors the status of cluster nodes.
func ( nc * NodeController ) Run ( ) {
go func ( ) {
defer utilruntime . HandleCrash ( )
if ! cache . WaitForCacheSync ( wait . NeverStop , nc . nodeInformer . Informer ( ) . HasSynced , nc . podInformer . Informer ( ) . HasSynced , nc . daemonSetInformer . Informer ( ) . HasSynced ) {
utilruntime . HandleError ( errors . New ( "NodeController timed out while waiting for informers to sync..." ) )
return
}
// Incorporate the results of node status pushed from kubelet to master.
go wait . Until ( func ( ) {
if err := nc . monitorNodeStatus ( ) ; err != nil {
glog . Errorf ( "Error monitoring node status: %v" , err )
}
} , nc . nodeMonitorPeriod , wait . NeverStop )
// Managing eviction of nodes:
// When we delete pods off a node, if the node was not empty at the time we then
// queue an eviction watcher. If we hit an error, retry deletion.
go wait . Until ( func ( ) {
nc . evictorLock . Lock ( )
defer nc . evictorLock . Unlock ( )
for k := range nc . zonePodEvictor {
nc . zonePodEvictor [ k ] . Try ( func ( value TimedValue ) ( bool , time . Duration ) {
obj , exists , err := nc . nodeStore . GetByKey ( value . Value )
if err != nil {
glog . Warningf ( "Failed to get Node %v from the nodeStore: %v" , value . Value , err )
} else if ! exists {
glog . Warningf ( "Node %v no longer present in nodeStore!" , value . Value )
} else {
node , _ := obj . ( * v1 . Node )
zone := utilnode . GetZoneKey ( node )
EvictionsNumber . WithLabelValues ( zone ) . Inc ( )
}
nodeUid , _ := value . UID . ( string )
remaining , err := deletePods ( nc . kubeClient , nc . recorder , value . Value , nodeUid , nc . daemonSetStore )
if err != nil {
utilruntime . HandleError ( fmt . Errorf ( "unable to evict node %q: %v" , value . Value , err ) )
return false , 0
}
if remaining {
glog . Infof ( "Pods awaiting deletion due to NodeController eviction" )
}
return true , 0
} )
}
} , nodeEvictionPeriod , wait . NeverStop )
} ( )
}
// monitorNodeStatus verifies node status are constantly updated by kubelet, and if not,
// post "NodeReady==ConditionUnknown". It also evicts all pods if node is not ready or
// not reachable for a long period of time.
func ( nc * NodeController ) monitorNodeStatus ( ) error {
// We are listing nodes from local cache as we can tolerate some small delays
// comparing to state from etcd and there is eventual consistency anyway.
nodes , err := nc . nodeStore . List ( )
if err != nil {
return err
}
added , deleted := nc . checkForNodeAddedDeleted ( & nodes )
for i := range added {
glog . V ( 1 ) . Infof ( "NodeController observed a new Node: %#v" , added [ i ] . Name )
recordNodeEvent ( nc . recorder , added [ i ] . Name , string ( added [ i ] . UID ) , v1 . EventTypeNormal , "RegisteredNode" , fmt . Sprintf ( "Registered Node %v in NodeController" , added [ i ] . Name ) )
nc . knownNodeSet [ added [ i ] . Name ] = added [ i ]
// When adding new Nodes we need to check if new zone appeared, and if so add new evictor.
zone := utilnode . GetZoneKey ( added [ i ] )
if _ , found := nc . zonePodEvictor [ zone ] ; ! found {
nc . zonePodEvictor [ zone ] =
NewRateLimitedTimedQueue (
flowcontrol . NewTokenBucketRateLimiter ( nc . evictionLimiterQPS , evictionRateLimiterBurst ) )
// Init the metric for the new zone.
glog . Infof ( "Initializing eviction metric for zone: %v" , zone )
EvictionsNumber . WithLabelValues ( zone ) . Add ( 0 )
}
nc . cancelPodEviction ( added [ i ] )
}
for i := range deleted {
glog . V ( 1 ) . Infof ( "NodeController observed a Node deletion: %v" , deleted [ i ] . Name )
recordNodeEvent ( nc . recorder , deleted [ i ] . Name , string ( deleted [ i ] . UID ) , v1 . EventTypeNormal , "RemovingNode" , fmt . Sprintf ( "Removing Node %v from NodeController" , deleted [ i ] . Name ) )
delete ( nc . knownNodeSet , deleted [ i ] . Name )
}
zoneToNodeConditions := map [ string ] [ ] * v1 . NodeCondition { }
for i := range nodes . Items {
var gracePeriod time . Duration
var observedReadyCondition v1 . NodeCondition
var currentReadyCondition * v1 . NodeCondition
nodeCopy , err := api . Scheme . DeepCopy ( & nodes . Items [ i ] )
if err != nil {
utilruntime . HandleError ( err )
continue
}
node := nodeCopy . ( * v1 . Node )
if err := wait . PollImmediate ( retrySleepTime , retrySleepTime * nodeStatusUpdateRetry , func ( ) ( bool , error ) {
gracePeriod , observedReadyCondition , currentReadyCondition , err = nc . tryUpdateNodeStatus ( node )
if err == nil {
return true , nil
}
name := node . Name
node , err = nc . kubeClient . Core ( ) . Nodes ( ) . Get ( name , metav1 . GetOptions { } )
if err != nil {
glog . Errorf ( "Failed while getting a Node to retry updating NodeStatus. Probably Node %s was deleted." , name )
return false , err
}
return false , nil
} ) ; err != nil {
glog . Errorf ( "Update status of Node %v from NodeController error : %v. " +
"Skipping - no pods will be evicted." , node . Name , err )
continue
}
// We do not treat a master node as a part of the cluster for network disruption checking.
if ! system . IsMasterNode ( node . Name ) {
zoneToNodeConditions [ utilnode . GetZoneKey ( node ) ] = append ( zoneToNodeConditions [ utilnode . GetZoneKey ( node ) ] , currentReadyCondition )
}
decisionTimestamp := nc . now ( )
if currentReadyCondition != nil {
// Check eviction timeout against decisionTimestamp
if observedReadyCondition . Status == v1 . ConditionFalse &&
decisionTimestamp . After ( nc . nodeStatusMap [ node . Name ] . readyTransitionTimestamp . Add ( nc . podEvictionTimeout ) ) {
if nc . evictPods ( node ) {
glog . V ( 2 ) . Infof ( "Evicting pods on node %s: %v is later than %v + %v" , node . Name , decisionTimestamp , nc . nodeStatusMap [ node . Name ] . readyTransitionTimestamp , nc . podEvictionTimeout )
}
}
if observedReadyCondition . Status == v1 . ConditionUnknown &&
decisionTimestamp . After ( nc . nodeStatusMap [ node . Name ] . probeTimestamp . Add ( nc . podEvictionTimeout ) ) {
if nc . evictPods ( node ) {
glog . V ( 2 ) . Infof ( "Evicting pods on node %s: %v is later than %v + %v" , node . Name , decisionTimestamp , nc . nodeStatusMap [ node . Name ] . readyTransitionTimestamp , nc . podEvictionTimeout - gracePeriod )
}
}
if observedReadyCondition . Status == v1 . ConditionTrue {
if nc . cancelPodEviction ( node ) {
glog . V ( 2 ) . Infof ( "Node %s is ready again, cancelled pod eviction" , node . Name )
}
}
// Report node event.
if currentReadyCondition . Status != v1 . ConditionTrue && observedReadyCondition . Status == v1 . ConditionTrue {
recordNodeStatusChange ( nc . recorder , node , "NodeNotReady" )
if err = markAllPodsNotReady ( nc . kubeClient , node ) ; err != nil {
utilruntime . HandleError ( fmt . Errorf ( "Unable to mark all pods NotReady on node %v: %v" , node . Name , err ) )
}
}
// Check with the cloud provider to see if the node still exists. If it
// doesn't, delete the node immediately.
if currentReadyCondition . Status != v1 . ConditionTrue && nc . cloud != nil {
exists , err := nc . nodeExistsInCloudProvider ( types . NodeName ( node . Name ) )
if err != nil {
glog . Errorf ( "Error determining if node %v exists in cloud: %v" , node . Name , err )
continue
}
if ! exists {
glog . V ( 2 ) . Infof ( "Deleting node (no longer present in cloud provider): %s" , node . Name )
recordNodeEvent ( nc . recorder , node . Name , string ( node . UID ) , v1 . EventTypeNormal , "DeletingNode" , fmt . Sprintf ( "Deleting Node %v because it's not present according to cloud provider" , node . Name ) )
go func ( nodeName string ) {
defer utilruntime . HandleCrash ( )
// Kubelet is not reporting and Cloud Provider says node
// is gone. Delete it without worrying about grace
// periods.
if err := forcefullyDeleteNode ( nc . kubeClient , nodeName ) ; err != nil {
glog . Errorf ( "Unable to forcefully delete node %q: %v" , nodeName , err )
}
} ( node . Name )
}
}
}
}
nc . handleDisruption ( zoneToNodeConditions , & nodes )
return nil
}
func ( nc * NodeController ) handleDisruption ( zoneToNodeConditions map [ string ] [ ] * v1 . NodeCondition , nodes * v1 . NodeList ) {
newZoneStates := map [ string ] zoneState { }
allAreFullyDisrupted := true
for k , v := range zoneToNodeConditions {
ZoneSize . WithLabelValues ( k ) . Set ( float64 ( len ( v ) ) )
unhealthy , newState := nc . computeZoneStateFunc ( v )
ZoneHealth . WithLabelValues ( k ) . Set ( float64 ( 100 * ( len ( v ) - unhealthy ) ) / float64 ( len ( v ) ) )
UnhealthyNodes . WithLabelValues ( k ) . Set ( float64 ( unhealthy ) )
if newState != stateFullDisruption {
allAreFullyDisrupted = false
}
newZoneStates [ k ] = newState
if _ , had := nc . zoneStates [ k ] ; ! had {
nc . zoneStates [ k ] = stateInitial
}
}
allWasFullyDisrupted := true
for k , v := range nc . zoneStates {
if _ , have := zoneToNodeConditions [ k ] ; ! have {
ZoneSize . WithLabelValues ( k ) . Set ( 0 )
ZoneHealth . WithLabelValues ( k ) . Set ( 100 )
UnhealthyNodes . WithLabelValues ( k ) . Set ( 0 )
delete ( nc . zoneStates , k )
continue
}
if v != stateFullDisruption {
allWasFullyDisrupted = false
break
}
}
// At least one node was responding in previous pass or in the current pass. Semantics is as follows:
// - if the new state is "partialDisruption" we call a user defined function that returns a new limiter to use,
// - if the new state is "normal" we resume normal operation (go back to default limiter settings),
// - if new state is "fullDisruption" we restore normal eviction rate,
// - unless all zones in the cluster are in "fullDisruption" - in that case we stop all evictions.
if ! allAreFullyDisrupted || ! allWasFullyDisrupted {
// We're switching to full disruption mode
if allAreFullyDisrupted {
glog . V ( 0 ) . Info ( "NodeController detected that all Nodes are not-Ready. Entering master disruption mode." )
for i := range nodes . Items {
nc . cancelPodEviction ( & nodes . Items [ i ] )
}
// We stop all evictions.
for k := range nc . zonePodEvictor {
nc . zonePodEvictor [ k ] . SwapLimiter ( 0 )
}
for k := range nc . zoneStates {
nc . zoneStates [ k ] = stateFullDisruption
}
// All rate limiters are updated, so we can return early here.
return
}
// We're exiting full disruption mode
if allWasFullyDisrupted {
glog . V ( 0 ) . Info ( "NodeController detected that some Nodes are Ready. Exiting master disruption mode." )
// When exiting disruption mode update probe timestamps on all Nodes.
now := nc . now ( )
for i := range nodes . Items {
v := nc . nodeStatusMap [ nodes . Items [ i ] . Name ]
v . probeTimestamp = now
v . readyTransitionTimestamp = now
nc . nodeStatusMap [ nodes . Items [ i ] . Name ] = v
}
// We reset all rate limiters to settings appropriate for the given state.
for k := range nc . zonePodEvictor {
nc . setLimiterInZone ( k , len ( zoneToNodeConditions [ k ] ) , newZoneStates [ k ] )
nc . zoneStates [ k ] = newZoneStates [ k ]
}
return
}
// We know that there's at least one not-fully disrupted so,
// we can use default behavior for rate limiters
for k , v := range nc . zoneStates {
newState := newZoneStates [ k ]
if v == newState {
continue
}
glog . V ( 0 ) . Infof ( "NodeController detected that zone %v is now in state %v." , k , newState )
nc . setLimiterInZone ( k , len ( zoneToNodeConditions [ k ] ) , newState )
nc . zoneStates [ k ] = newState
}
}
}
func ( nc * NodeController ) setLimiterInZone ( zone string , zoneSize int , state zoneState ) {
switch state {
case stateNormal :
nc . zonePodEvictor [ zone ] . SwapLimiter ( nc . evictionLimiterQPS )
case statePartialDisruption :
nc . zonePodEvictor [ zone ] . SwapLimiter (
nc . enterPartialDisruptionFunc ( zoneSize ) )
case stateFullDisruption :
nc . zonePodEvictor [ zone ] . SwapLimiter (
nc . enterFullDisruptionFunc ( zoneSize ) )
}
}
// For a given node checks its conditions and tries to update it. Returns grace period to which given node
// is entitled, state of current and last observed Ready Condition, and an error if it occurred.
func ( nc * NodeController ) tryUpdateNodeStatus ( node * v1 . Node ) ( time . Duration , v1 . NodeCondition , * v1 . NodeCondition , error ) {
var err error
var gracePeriod time . Duration
var observedReadyCondition v1 . NodeCondition
_ , currentReadyCondition := v1 . GetNodeCondition ( & node . Status , v1 . NodeReady )
if currentReadyCondition == nil {
// If ready condition is nil, then kubelet (or nodecontroller) never posted node status.
// A fake ready condition is created, where LastProbeTime and LastTransitionTime is set
// to node.CreationTimestamp to avoid handle the corner case.
observedReadyCondition = v1 . NodeCondition {
Type : v1 . NodeReady ,
Status : v1 . ConditionUnknown ,
LastHeartbeatTime : node . CreationTimestamp ,
LastTransitionTime : node . CreationTimestamp ,
}
gracePeriod = nc . nodeStartupGracePeriod
nc . nodeStatusMap [ node . Name ] = nodeStatusData {
status : node . Status ,
probeTimestamp : node . CreationTimestamp ,
readyTransitionTimestamp : node . CreationTimestamp ,
}
} else {
// If ready condition is not nil, make a copy of it, since we may modify it in place later.
observedReadyCondition = * currentReadyCondition
gracePeriod = nc . nodeMonitorGracePeriod
}
savedNodeStatus , found := nc . nodeStatusMap [ node . Name ]
// There are following cases to check:
// - both saved and new status have no Ready Condition set - we leave everything as it is,
// - saved status have no Ready Condition, but current one does - NodeController was restarted with Node data already present in etcd,
// - saved status have some Ready Condition, but current one does not - it's an error, but we fill it up because that's probably a good thing to do,
// - both saved and current statuses have Ready Conditions and they have the same LastProbeTime - nothing happened on that Node, it may be
// unresponsive, so we leave it as it is,
// - both saved and current statuses have Ready Conditions, they have different LastProbeTimes, but the same Ready Condition State -
// everything's in order, no transition occurred, we update only probeTimestamp,
// - both saved and current statuses have Ready Conditions, different LastProbeTimes and different Ready Condition State -
// Ready Condition changed it state since we last seen it, so we update both probeTimestamp and readyTransitionTimestamp.
// TODO: things to consider:
// - if 'LastProbeTime' have gone back in time its probably an error, currently we ignore it,
// - currently only correct Ready State transition outside of Node Controller is marking it ready by Kubelet, we don't check
// if that's the case, but it does not seem necessary.
var savedCondition * v1 . NodeCondition
if found {
_ , savedCondition = v1 . GetNodeCondition ( & savedNodeStatus . status , v1 . NodeReady )
}
_ , observedCondition := v1 . GetNodeCondition ( & node . Status , v1 . NodeReady )
if ! found {
glog . Warningf ( "Missing timestamp for Node %s. Assuming now as a timestamp." , node . Name )
savedNodeStatus = nodeStatusData {
status : node . Status ,
probeTimestamp : nc . now ( ) ,
readyTransitionTimestamp : nc . now ( ) ,
}
} else if savedCondition == nil && observedCondition != nil {
glog . V ( 1 ) . Infof ( "Creating timestamp entry for newly observed Node %s" , node . Name )
savedNodeStatus = nodeStatusData {
status : node . Status ,
probeTimestamp : nc . now ( ) ,
readyTransitionTimestamp : nc . now ( ) ,
}
} else if savedCondition != nil && observedCondition == nil {
glog . Errorf ( "ReadyCondition was removed from Status of Node %s" , node . Name )
// TODO: figure out what to do in this case. For now we do the same thing as above.
savedNodeStatus = nodeStatusData {
status : node . Status ,
probeTimestamp : nc . now ( ) ,
readyTransitionTimestamp : nc . now ( ) ,
}
} else if savedCondition != nil && observedCondition != nil && savedCondition . LastHeartbeatTime != observedCondition . LastHeartbeatTime {
var transitionTime metav1 . Time
// If ReadyCondition changed since the last time we checked, we update the transition timestamp to "now",
// otherwise we leave it as it is.
if savedCondition . LastTransitionTime != observedCondition . LastTransitionTime {
glog . V ( 3 ) . Infof ( "ReadyCondition for Node %s transitioned from %v to %v" , node . Name , savedCondition . Status , observedCondition )
transitionTime = nc . now ( )
} else {
transitionTime = savedNodeStatus . readyTransitionTimestamp
}
if glog . V ( 5 ) {
glog . V ( 5 ) . Infof ( "Node %s ReadyCondition updated. Updating timestamp: %+v vs %+v." , node . Name , savedNodeStatus . status , node . Status )
} else {
glog . V ( 3 ) . Infof ( "Node %s ReadyCondition updated. Updating timestamp." , node . Name )
}
savedNodeStatus = nodeStatusData {
status : node . Status ,
probeTimestamp : nc . now ( ) ,
readyTransitionTimestamp : transitionTime ,
}
}
nc . nodeStatusMap [ node . Name ] = savedNodeStatus
if nc . now ( ) . After ( savedNodeStatus . probeTimestamp . Add ( gracePeriod ) ) {
// NodeReady condition was last set longer ago than gracePeriod, so update it to Unknown
// (regardless of its current value) in the master.
if currentReadyCondition == nil {
glog . V ( 2 ) . Infof ( "node %v is never updated by kubelet" , node . Name )
node . Status . Conditions = append ( node . Status . Conditions , v1 . NodeCondition {
Type : v1 . NodeReady ,
Status : v1 . ConditionUnknown ,
Reason : "NodeStatusNeverUpdated" ,
Message : fmt . Sprintf ( "Kubelet never posted node status." ) ,
LastHeartbeatTime : node . CreationTimestamp ,
LastTransitionTime : nc . now ( ) ,
} )
} else {
glog . V ( 4 ) . Infof ( "node %v hasn't been updated for %+v. Last ready condition is: %+v" ,
node . Name , nc . now ( ) . Time . Sub ( savedNodeStatus . probeTimestamp . Time ) , observedReadyCondition )
if observedReadyCondition . Status != v1 . ConditionUnknown {
currentReadyCondition . Status = v1 . ConditionUnknown
currentReadyCondition . Reason = "NodeStatusUnknown"
currentReadyCondition . Message = fmt . Sprintf ( "Kubelet stopped posting node status." )
// LastProbeTime is the last time we heard from kubelet.
currentReadyCondition . LastHeartbeatTime = observedReadyCondition . LastHeartbeatTime
currentReadyCondition . LastTransitionTime = nc . now ( )
}
}
// Like NodeReady condition, NodeOutOfDisk was last set longer ago than gracePeriod, so update
// it to Unknown (regardless of its current value) in the master.
// TODO(madhusudancs): Refactor this with readyCondition to remove duplicated code.
_ , oodCondition := v1 . GetNodeCondition ( & node . Status , v1 . NodeOutOfDisk )
if oodCondition == nil {
glog . V ( 2 ) . Infof ( "Out of disk condition of node %v is never updated by kubelet" , node . Name )
node . Status . Conditions = append ( node . Status . Conditions , v1 . NodeCondition {
Type : v1 . NodeOutOfDisk ,
Status : v1 . ConditionUnknown ,
Reason : "NodeStatusNeverUpdated" ,
Message : fmt . Sprintf ( "Kubelet never posted node status." ) ,
LastHeartbeatTime : node . CreationTimestamp ,
LastTransitionTime : nc . now ( ) ,
} )
} else {
glog . V ( 4 ) . Infof ( "node %v hasn't been updated for %+v. Last out of disk condition is: %+v" ,
node . Name , nc . now ( ) . Time . Sub ( savedNodeStatus . probeTimestamp . Time ) , oodCondition )
if oodCondition . Status != v1 . ConditionUnknown {
oodCondition . Status = v1 . ConditionUnknown
oodCondition . Reason = "NodeStatusUnknown"
oodCondition . Message = fmt . Sprintf ( "Kubelet stopped posting node status." )
oodCondition . LastTransitionTime = nc . now ( )
}
}
_ , currentCondition := v1 . GetNodeCondition ( & node . Status , v1 . NodeReady )
if ! api . Semantic . DeepEqual ( currentCondition , & observedReadyCondition ) {
if _ , err = nc . kubeClient . Core ( ) . Nodes ( ) . UpdateStatus ( node ) ; err != nil {
glog . Errorf ( "Error updating node %s: %v" , node . Name , err )
return gracePeriod , observedReadyCondition , currentReadyCondition , err
} else {
nc . nodeStatusMap [ node . Name ] = nodeStatusData {
status : node . Status ,
probeTimestamp : nc . nodeStatusMap [ node . Name ] . probeTimestamp ,
readyTransitionTimestamp : nc . now ( ) ,
}
return gracePeriod , observedReadyCondition , currentReadyCondition , nil
}
}
}
return gracePeriod , observedReadyCondition , currentReadyCondition , err
}
func ( nc * NodeController ) checkForNodeAddedDeleted ( nodes * v1 . NodeList ) ( added , deleted [ ] * v1 . Node ) {
for i := range nodes . Items {
if _ , has := nc . knownNodeSet [ nodes . Items [ i ] . Name ] ; ! has {
added = append ( added , & nodes . Items [ i ] )
}
}
// If there's a difference between lengths of known Nodes and observed nodes
// we must have removed some Node.
if len ( nc . knownNodeSet ) + len ( added ) != len ( nodes . Items ) {
knowSetCopy := map [ string ] * v1 . Node { }
for k , v := range nc . knownNodeSet {
knowSetCopy [ k ] = v
}
for i := range nodes . Items {
delete ( knowSetCopy , nodes . Items [ i ] . Name )
}
for i := range knowSetCopy {
deleted = append ( deleted , knowSetCopy [ i ] )
}
}
return
}
// cancelPodEviction removes any queued evictions, typically because the node is available again. It
// returns true if an eviction was queued.
func ( nc * NodeController ) cancelPodEviction ( node * v1 . Node ) bool {
zone := utilnode . GetZoneKey ( node )
nc . evictorLock . Lock ( )
defer nc . evictorLock . Unlock ( )
wasDeleting := nc . zonePodEvictor [ zone ] . Remove ( node . Name )
if wasDeleting {
glog . V ( 2 ) . Infof ( "Cancelling pod Eviction on Node: %v" , node . Name )
return true
}
return false
}
// evictPods queues an eviction for the provided node name, and returns false if the node is already
// queued for eviction.
func ( nc * NodeController ) evictPods ( node * v1 . Node ) bool {
nc . evictorLock . Lock ( )
defer nc . evictorLock . Unlock ( )
return nc . zonePodEvictor [ utilnode . GetZoneKey ( node ) ] . Add ( node . Name , string ( node . UID ) )
}
// Default value for cluster eviction rate - we take nodeNum for consistency with ReducedQPSFunc.
func ( nc * NodeController ) HealthyQPSFunc ( nodeNum int ) float32 {
return nc . evictionLimiterQPS
}
// If the cluster is large make evictions slower, if they're small stop evictions altogether.
func ( nc * NodeController ) ReducedQPSFunc ( nodeNum int ) float32 {
if int32 ( nodeNum ) > nc . largeClusterThreshold {
return nc . secondaryEvictionLimiterQPS
}
return 0
}
// This function is expected to get a slice of NodeReadyConditions for all Nodes in a given zone.
// The zone is considered:
// - fullyDisrupted if there're no Ready Nodes,
// - partiallyDisrupted if at least than nc.unhealthyZoneThreshold percent of Nodes are not Ready,
// - normal otherwise
func ( nc * NodeController ) ComputeZoneState ( nodeReadyConditions [ ] * v1 . NodeCondition ) ( int , zoneState ) {
readyNodes := 0
notReadyNodes := 0
for i := range nodeReadyConditions {
if nodeReadyConditions [ i ] != nil && nodeReadyConditions [ i ] . Status == v1 . ConditionTrue {
readyNodes ++
} else {
notReadyNodes ++
}
}
switch {
case readyNodes == 0 && notReadyNodes > 0 :
return notReadyNodes , stateFullDisruption
case notReadyNodes > 2 && float32 ( notReadyNodes ) / float32 ( notReadyNodes + readyNodes ) >= nc . unhealthyZoneThreshold :
return notReadyNodes , statePartialDisruption
default :
return notReadyNodes , stateNormal
}
}