Merge pull request #65 from dougbtv/backport-readiness-check-cni-del-43

Bug 1824940: Adds readinessindicatorfile check on CNI DEL

doc/how-to-use.md

@@ -619,3 +619,7 @@ When using `--multus-conf-file=auto` you may also care to specify a `binDir` in
 Sometimes, you may wish to not have the entrypoint copy the binary file onto the host. Potentially, you have another way to copy in a specific version of Multus, for example. By default, it's always copied, but you may disable the copy with:
 
     --skip-multus-binary-copy=true
+
+If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
+
+    --readiness-indicator-file=/path/to/file

images/entrypoint.sh

@@ -3,20 +3,6 @@
 # Always exit on errors.
 set -e
 
-# Run a clean up when we exit if configured to do so.
-trap cleanup TERM
-function cleanup {
-  if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == "true" ]; then
-    CONF=$(cat <<-EOF
-        {Multus configuration intentionally invalidated to prevent pods from being scheduled.}
-EOF
-      )
-      echo $CONF > $CNI_CONF_DIR/00-multus.conf
-      log "Multus configuration intentionally invalidated to prevent pods from being scheduled."
-  fi
-}
-
-
 # Set our known directories.
 CNI_CONF_DIR="/host/etc/cni/net.d"
 CNI_BIN_DIR="/host/opt/cni/bin"
@@ -28,6 +14,7 @@ MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
 MULTUS_NAMESPACE_ISOLATION=false
 MULTUS_LOG_LEVEL=""
 MULTUS_LOG_FILE=""
+MULTUS_READINESS_INDICATOR_FILE=""
 OVERRIDE_NETWORK_NAME=false
 MULTUS_CLEANUP_CONFIG_ON_EXIT=false
 RESTART_CRIO=false
@@ -61,6 +48,7 @@ function usage()
     echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
     echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)"
     echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--readiness-indicator-file=$MULTUS_READINESS_INDICATOR_FILE (used only with --multus-conf-file=auto)"
     echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)"
     echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)"
 }
@@ -137,6 +125,9 @@ while [ "$1" != "" ]; do
         --skip-multus-binary-copy)
             SKIP_BINARY_COPY=$VALUE
             ;;
+        --readiness-indicator-file)
+            MULTUS_READINESS_INDICATOR_FILE=$VALUE
+            ;;
         *)
             warn "unknown parameter \"$PARAM\""
             ;;
@@ -250,13 +241,6 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
           log "Attemping to find master plugin configuration, attempt $tries"
         fi
         let "tries+=1"
-        # See if the Multus configuration file exists, if it does then clean it up.
-        if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ] && [ -f "$CNI_CONF_DIR/00-multus.conf" ]; then
-          # But first, check if it has the invalidated configuration in it (otherwise we keep doing this over and over.)
-          if ! grep -q "invalidated" $CNI_CONF_DIR/00-multus.conf; then
-            cleanup
-          fi
-        fi
         sleep 1;
       else
         error "Multus could not be configured: no master plugin was found."
@@ -305,6 +289,12 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
         ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\","
       fi
 
+
+      READINESS_INDICATOR_FILE_STRING=""
+      if [ ! -z "${MULTUS_READINESS_INDICATOR_FILE// }" ]; then
+        READINESS_INDICATOR_FILE_STRING="\"readinessindicatorfile\": \"$MULTUS_READINESS_INDICATOR_FILE\","
+      fi
+
       if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
         MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
             python -c 'import json,sys;print json.load(sys.stdin)["name"]')"
@@ -324,6 +314,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then
           $LOG_LEVEL_STRING
           $LOG_FILE_STRING
           $ADDITIONAL_BIN_DIR_STRING
+          $READINESS_INDICATOR_FILE_STRING
           "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
           "delegates": [
             $MASTER_PLUGIN_JSON
@@ -363,8 +354,17 @@ if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
   while true; do
     # Check and see if the original master plugin configuration exists...
     if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then
-      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Performing cleanup..."
+      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Allowing 45 seconds for its restoration..."
-      cleanup
+      sleep 10
+      for i in {1..35}
+      do
+        if [ -f "$MASTER_PLUGIN_LOCATION" ]; then
+          log "Master plugin @ $MASTER_PLUGIN_LOCATION was restored. Regenerating given configuration."
+          break
+        fi
+        sleep 1
+      done
+
       generateMultusConf
       log "Continuing watch loop after configuration regeneration..."
     fi

multus/multus.go

@@ -48,12 +48,8 @@ var version = "master@git"
 var commit = "unknown commit"
 var date = "unknown date"
 
-var defaultReadinessBackoff = wait.Backoff{
+var pollDuration = 1000 * time.Millisecond
-	Steps:    4,
+var pollTimeout = 45 * time.Second
-	Duration: 250 * time.Millisecond,
-	Factor:   4.0,
-	Jitter:   0.1,
-}
 
 func printVersionString() string {
 	return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s",
@@ -359,32 +355,40 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN
 	return nil
 }
 
+func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error {
+	prefix := "Multus: "
+	if k8sArgs != nil {
+		prefix += fmt.Sprintf("[%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME)
+	}
+	return logging.Errorf(prefix+format, args...)
+}
+
 func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) {
 	n, err := types.LoadNetConf(args.StdinData)
 	logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
-		return nil, logging.Errorf("Multus: error loading netconf: %v", err)
+		return nil, cmdErr(nil, "error loading netconf: %v", err)
 	}
 
 	k8sArgs, err := k8s.GetK8sArgs(args)
 	if err != nil {
-		return nil, logging.Errorf("Multus: error getting k8s args: %v", err)
+		return nil, cmdErr(nil, "error getting k8s args: %v", err)
 	}
 
-	wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) {
+	if n.ReadinessIndicatorFile != "" {
+		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
 			_, err := os.Stat(n.ReadinessIndicatorFile)
-		switch {
+			return err == nil, nil
-		case err == nil:
-			return true, nil
-		default:
-			return false, nil
-		}
 		})
+		if err != nil {
+			return nil, cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile: %v", err)
+		}
+	}
 
 	if n.ClusterNetwork != "" {
 		err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient)
 		if err != nil {
-			return nil, logging.Errorf("Multus: failed to get clusterNetwork/defaultNetworks: %v", err)
+			return nil, cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err)
 		}
 		// First delegate is always the master plugin
 		n.Delegates[0].MasterPlugin = true
@@ -392,12 +396,12 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 
 	_, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient)
 	if err != nil {
-		return nil, logging.Errorf("Multus: error loading k8s delegates k8s args: %v", err)
+		return nil, cmdErr(k8sArgs, "error loading k8s delegates k8s args: %v", err)
 	}
 
 	// cache the multus config
 	if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
-		return nil, logging.Errorf("Multus: error saving the delegates: %v", err)
+		return nil, cmdErr(k8sArgs, "error saving the delegates: %v", err)
 	}
 
 	var result, tmpResult cnitypes.Result
@@ -417,7 +421,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 			}
 			// Ignore errors; DEL must be idempotent anyway
 			_ = delPlugins(exec, args.IfName, n.Delegates, idx, rt, n.BinDir)
-			return nil, logging.Errorf("Multus: error adding pod to network %q: %v", netName, err)
+			return nil, cmdErr(k8sArgs, "error adding container to network %q: %v", netName, err)
 		}
 
 		// Remove gateway from routing table if the gateway is not used
@@ -438,7 +442,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		if deletegateway {
 			tmpResult, err = netutils.DeleteDefaultGW(args, ifName, &tmpResult)
 			if err != nil {
-				return nil, logging.Errorf("Multus: Err in deleting gateway: %v", err)
+				return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err)
 			}
 		}
 
@@ -446,7 +450,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		if adddefaultgateway {
 			tmpResult, err = netutils.SetDefaultGW(args, ifName, delegate.GatewayRequest, &tmpResult)
 			if err != nil {
-				return nil, logging.Errorf("Multus: Err in setting default gateway: %v", err)
+				return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err)
 			}
 		}
 
@@ -460,7 +464,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 			if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) {
 				delegateNetStatus, err := types.LoadNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin)
 				if err != nil {
-					return nil, logging.Errorf("Multus: error setting network status: %v", err)
+					return nil, cmdErr(k8sArgs, "error setting network status: %v", err)
 				}
 
 				netStatus = append(netStatus, delegateNetStatus)
@@ -473,7 +477,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) {
 			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
-				return nil, logging.Errorf("Multus: error setting the networks status: %v", err)
+				return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err)
 			}
 		}
 	}
@@ -511,7 +515,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 			netnsfound = false
 			logging.Debugf("cmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err)
 		} else {
-			return logging.Errorf("Multus: failed to open netns %q: %v", netns, err)
+			return cmdErr(nil, "failed to open netns %q: %v", netns, err)
 		}
 	}
 
@@ -521,7 +525,17 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 
 	k8sArgs, err := k8s.GetK8sArgs(args)
 	if err != nil {
-		return logging.Errorf("Multus: error getting k8s args: %v", err)
+		return cmdErr(nil, "error getting k8s args: %v", err)
+	}
+
+	if in.ReadinessIndicatorFile != "" {
+		err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
+			_, err := os.Stat(in.ReadinessIndicatorFile)
+			return err == nil, nil
+		})
+		if err != nil {
+			return cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile (on del): %v", err)
+		}
 	}
 
 	// Read the cache to get delegates json for the pod
@@ -532,7 +546,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 			if in.ClusterNetwork != "" {
 				err = k8s.GetDefaultNetworks(k8sArgs, in, kubeClient)
 				if err != nil {
-					return logging.Errorf("Multus: failed to get clusterNetwork/defaultNetworks: %v", err)
+					return cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err)
 				}
 				// First delegate is always the master plugin
 				in.Delegates[0].MasterPlugin = true
@@ -543,18 +557,18 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 			if err != nil {
 				if len(in.Delegates) == 0 {
 					// No delegate available so send error
-					return logging.Errorf("Multus: failed to get delegates: %v", err)
+					return cmdErr(k8sArgs, "failed to get delegates: %v", err)
 				}
 				// Get clusterNetwork before, so continue to delete
 				logging.Errorf("Multus: failed to get delegates: %v, but continue to delete clusterNetwork", err)
 			}
 		} else {
-			return logging.Errorf("Multus: error reading the delegates: %v", err)
+			return cmdErr(k8sArgs, "error reading the delegates: %v", err)
 		}
 	} else {
 		defer os.Remove(path)
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
-			return logging.Errorf("Multus: failed to load netconf: %v", err)
+			return cmdErr(k8sArgs, "failed to load netconf: %v", err)
 		}
 		// check plugins field and enable ConfListPlugin if there is
 		for _, v := range in.Delegates {