ocicni: Support asynchronous network config creation

We need to support cases where InitCNI() is called before
any CNI configuration files have been installed. This is
for example happening when deploying a k8s cluster with kubeadm.
kubeadm will start the DNS pod and it is left to the caller to
pick a network overlay and create the corresponding pods, that
will typically install a CNI configuration file first.

Here we address that issue by doing 2 things:

- Not returning an error when the default CNI config files
  directory is empty.
- If it is empty, we start a monitoring thread (fsnotify based)
  that will synchronize the network configuration when a CNI
  file is installed there.

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

pkg/ocicni/ocicni.go

@@ -6,11 +6,11 @@ import (
 	"os/exec"
 	"sort"
 	"sync"
-	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/containernetworking/cni/libcni"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/fsnotify/fsnotify"
 )
 
 type cniNetworkPlugin struct {
@@ -23,6 +23,8 @@ type cniNetworkPlugin struct {
 	pluginDir          string
 	cniDirs            []string
 	vendorCNIDirPrefix string
+
+	monitorNetDirChan chan struct{}
 }
 
 type cniNetwork struct {
@@ -31,6 +33,49 @@ type cniNetwork struct {
 	CNIConfig     libcni.CNI
 }
 
+var errMissingDefaultNetwork = errors.New("Missing CNI default network")
+
+func (plugin *cniNetworkPlugin) monitorNetDir() {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		logrus.Errorf("could not create new watcher %v", err)
+		return
+	}
+	defer watcher.Close()
+
+	go func() {
+		for {
+			select {
+			case event := <-watcher.Events:
+				logrus.Debugf("CNI monitoring event %v", event)
+				if event.Op&fsnotify.Create != fsnotify.Create {
+					continue
+				}
+
+				if err = plugin.syncNetworkConfig(); err == nil {
+					logrus.Debugf("CNI asynchronous setting succeeded")
+					close(plugin.monitorNetDirChan)
+					return
+				}
+
+				logrus.Errorf("CNI setting failed, continue monitoring: %v", err)
+
+			case err := <-watcher.Errors:
+				logrus.Errorf("CNI monitoring error %v", err)
+				close(plugin.monitorNetDirChan)
+				return
+			}
+		}
+	}()
+
+	if err = watcher.Add(plugin.pluginDir); err != nil {
+		logrus.Error(err)
+		return
+	}
+
+	<-plugin.monitorNetDirChan
+}
+
 // InitCNI takes the plugin directory and cni directories where the cni files should be searched for
 // Returns a valid plugin object and any error
 func InitCNI(pluginDir string, cniDirs ...string) (CNIPlugin, error) {
@@ -44,19 +89,16 @@ func InitCNI(pluginDir string, cniDirs ...string) (CNIPlugin, error) {
 	// check if a default network exists, otherwise dump the CNI search and return a noop plugin
 	_, err = getDefaultCNINetwork(plugin.pluginDir, plugin.cniDirs, plugin.vendorCNIDirPrefix)
 	if err != nil {
+		if err != errMissingDefaultNetwork {
 			logrus.Warningf("Error in finding usable CNI plugin - %v", err)
 			// create a noop plugin instead
 			return &cniNoOp{}, nil
 		}
 
-	// sync network config from pluginDir periodically to detect network config updates
-	go func() {
-		t := time.NewTimer(10 * time.Second)
-		for {
-			plugin.syncNetworkConfig()
-			<-t.C
+		// We do not have a default network, we start the monitoring thread.
+		go plugin.monitorNetDir()
 	}
-	}()
+
 	return plugin, nil
 }
 
@@ -67,10 +109,13 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir string, cniDirs []strin
 		pluginDir:          pluginDir,
 		cniDirs:            cniDirs,
 		vendorCNIDirPrefix: vendorCNIDirPrefix,
+		monitorNetDirChan:  make(chan struct{}),
 	}
 
 	// sync NetworkConfig in best effort during probing.
-	plugin.syncNetworkConfig()
+	if err := plugin.syncNetworkConfig(); err != nil {
+		logrus.Error(err)
+	}
 	return plugin
 }
 
@@ -87,7 +132,7 @@ func getDefaultCNINetwork(pluginDir string, cniDirs []string, vendorCNIDirPrefix
 	case err != nil:
 		return nil, err
 	case len(files) == 0:
-		return nil, fmt.Errorf("No networks found in %s", pluginDir)
+		return nil, errMissingDefaultNetwork
 	}
 
 	sort.Strings(files)
@@ -142,13 +187,15 @@ func getLoNetwork(cniDirs []string, vendorDirPrefix string) *cniNetwork {
 	return loNetwork
 }
 
-func (plugin *cniNetworkPlugin) syncNetworkConfig() {
+func (plugin *cniNetworkPlugin) syncNetworkConfig() error {
 	network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.cniDirs, plugin.vendorCNIDirPrefix)
 	if err != nil {
 		logrus.Errorf("error updating cni config: %s", err)
-		return
+		return err
 	}
 	plugin.setDefaultNetwork(network)
+
+	return nil
 }
 
 func (plugin *cniNetworkPlugin) getDefaultNetwork() *cniNetwork {