Merge pull request #32151 from bboreham/fix-cni-on-gci

Automatic merge from submit-queue

Add flag to set CNI bin dir, and use it on gci nodes

**What this PR does / why we need it**:

When using `kube-up` on GCE, following #31023 which moved the workers from debian to gci, CNI just isn't working.  The root cause is basically as discussed in #28563: one flag (`--network-plugin-dir`) means two different things, and the `configure-helper` script uses it for the wrong purpose.

This PR adds a new flag `--cni-bin-dir`, then uses it to configure CNI as desired.

As discussed at #28563, I have also added a flag `--cni-conf-dir` so users can be explicit 

**Which issue this PR fixes** : fixes #28563

**Special notes for your reviewer**:

I left the old flag largely alone for backwards-compatibility, with the exception that I stop setting the default when CNI is in use.  The value of `"/usr/libexec/kubernetes/kubelet-plugins/net/exec/"` is unlikely to be what is wanted there.

**Release note**:
```release-note
Added new kubelet flags `--cni-bin-dir` and `--cni-conf-dir` to specify where CNI files are located.
Fixed CNI configuration on GCI platform when using CNI.
```

cluster/gce/gci/configure-helper.sh

@@ -496,7 +496,11 @@ function start-kubelet {
   fi
   # Network plugin
   if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
-    flags+=" --network-plugin-dir=/home/kubernetes/bin"
+    if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
+      flags+=" --cni-bin-dir=/home/kubernetes/bin"
+    else
+      flags+=" --network-plugin-dir=/home/kubernetes/bin"
+    fi
     flags+=" --network-plugin=${NETWORK_PROVIDER}"
   fi
   flags+=" --reconcile-cidr=${reconcile_cidr}"

cmd/kubelet/app/options/options.go

@@ -147,7 +147,9 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", s.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected. Default: 256")
 	fs.DurationVar(&s.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", s.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.  Default: '1m'")
 	fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
-	fs.StringVar(&s.NetworkPluginDir, "network-plugin-dir", s.NetworkPluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for network plugins")
+	fs.StringVar(&s.NetworkPluginDir, "network-plugin-dir", s.NetworkPluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for network plugins or CNI config")
+	fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d")
+	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin")
 	fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, "<Warning: Alpha feature> The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.")
 	fs.StringVar(&s.VolumePluginDir, "volume-plugin-dir", s.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider. [default=auto-detect]")

cmd/kubelet/app/plugins.go

@@ -90,12 +90,16 @@ func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin {
 }
 
 // ProbeNetworkPlugins collects all compiled-in plugins
-func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
+func ProbeNetworkPlugins(pluginDir, cniConfDir, cniBinDir string) []network.NetworkPlugin {
 	allPlugins := []network.NetworkPlugin{}
 
+	// for backwards-compat, allow pluginDir as a source of CNI config files
+	if cniConfDir == "" {
+		cniConfDir = pluginDir
+	}
 	// for each existing plugin, add to the list
 	allPlugins = append(allPlugins, exec.ProbeNetworkPlugins(pluginDir)...)
-	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(pluginDir)...)
+	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...)
 	allPlugins = append(allPlugins, kubenet.NewPlugin(pluginDir))
 
 	return allPlugins

cmd/kubelet/app/server.go

@@ -139,7 +139,7 @@ func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error
 		DockerClient:      dockerClient,
 		KubeClient:        nil,
 		Mounter:           mounter,
-		NetworkPlugins:    ProbeNetworkPlugins(s.NetworkPluginDir),
+		NetworkPlugins:    ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir),
 		OOMAdjuster:       oom.NewOOMAdjuster(),
 		OSInterface:       kubecontainer.RealOS{},
 		Writer:            writer,

hack/verify-flags/known-flags.txt

@@ -71,6 +71,8 @@ cluster-tag
 cluster-monitor-period
 cluster-signing-cert-file
 cluster-signing-key-file
+cni-bin-dir
+cni-conf-dir
 concurrent-deployment-syncs
 concurrent-endpoint-syncs
 concurrent-namespace-syncs

pkg/apis/componentconfig/types.go

@@ -257,8 +257,14 @@ type KubeletConfiguration struct {
 	// computed (such as IPSEC).
 	NetworkPluginMTU int32 `json:"networkPluginMTU"`
 	// networkPluginDir is the full path of the directory in which to search
-	// for network plugins
+	// for network plugins (and, for backwards-compat, CNI config files)
 	NetworkPluginDir string `json:"networkPluginDir"`
+	// CNIConfDir is the full path of the directory in which to search for
+	// CNI config files
+	CNIConfDir string `json:"cniConfDir"`
+	// CNIBinDir is the full path of the directory in which to search for
+	// CNI plugin binaries
+	CNIBinDir string `json:"cniBinDir"`
 	// volumePluginDir is the full path of the directory in which to search
 	// for additional third party volume plugins
 	VolumePluginDir string `json:"volumePluginDir"`

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -249,9 +249,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.MinimumGCAge == zeroDuration {
 		obj.MinimumGCAge = unversioned.Duration{Duration: 0}
 	}
-	if obj.NetworkPluginDir == "" {
-		obj.NetworkPluginDir = "/usr/libexec/kubernetes/kubelet-plugins/net/exec/"
-	}
 	if obj.NonMasqueradeCIDR == "" {
 		obj.NonMasqueradeCIDR = "10.0.0.0/8"
 	}

pkg/apis/componentconfig/v1alpha1/types.go

@@ -308,8 +308,14 @@ type KubeletConfiguration struct {
 	// various events in kubelet/pod lifecycle
 	NetworkPluginName string `json:"networkPluginName"`
 	// networkPluginDir is the full path of the directory in which to search
-	// for network plugins
+	// for network plugins (and, for backwards-compat, CNI config files)
 	NetworkPluginDir string `json:"networkPluginDir"`
+	// CNIConfDir is the full path of the directory in which to search for
+	// CNI config files
+	CNIConfDir string `json:"cniConfDir"`
+	// CNIBinDir is the full path of the directory in which to search for
+	// CNI plugin binaries
+	CNIBinDir string `json:"cniBinDir"`
 	// networkPluginMTU is the MTU to be passed to the network plugin,
 	// and overrides the default MTU for cases where it cannot be automatically
 	// computed (such as IPSEC).

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -233,6 +233,8 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod
 	out.NetworkPluginName = in.NetworkPluginName
 	out.NetworkPluginDir = in.NetworkPluginDir
+	out.CNIConfDir = in.CNIConfDir
+	out.CNIBinDir = in.CNIBinDir
 	out.NetworkPluginMTU = in.NetworkPluginMTU
 	out.VolumePluginDir = in.VolumePluginDir
 	out.CloudProvider = in.CloudProvider
@@ -412,6 +414,8 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	out.NetworkPluginName = in.NetworkPluginName
 	out.NetworkPluginMTU = in.NetworkPluginMTU
 	out.NetworkPluginDir = in.NetworkPluginDir
+	out.CNIConfDir = in.CNIConfDir
+	out.CNIBinDir = in.CNIBinDir
 	out.VolumePluginDir = in.VolumePluginDir
 	out.CloudProvider = in.CloudProvider
 	out.CloudConfigFile = in.CloudConfigFile

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -239,6 +239,8 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod
 		out.NetworkPluginName = in.NetworkPluginName
 		out.NetworkPluginDir = in.NetworkPluginDir
+		out.CNIConfDir = in.CNIConfDir
+		out.CNIBinDir = in.CNIBinDir
 		out.NetworkPluginMTU = in.NetworkPluginMTU
 		out.VolumePluginDir = in.VolumePluginDir
 		out.CloudProvider = in.CloudProvider

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -259,6 +259,8 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		out.NetworkPluginName = in.NetworkPluginName
 		out.NetworkPluginMTU = in.NetworkPluginMTU
 		out.NetworkPluginDir = in.NetworkPluginDir
+		out.CNIConfDir = in.CNIConfDir
+		out.CNIBinDir = in.CNIBinDir
 		out.VolumePluginDir = in.VolumePluginDir
 		out.CloudProvider = in.CloudProvider
 		out.CloudConfigFile = in.CloudConfigFile

pkg/kubelet/network/cni/cni.go

@@ -52,6 +52,7 @@ type cniNetworkPlugin struct {
 	execer             utilexec.Interface
 	nsenterPath        string
 	pluginDir          string
+	binDir             string
 	vendorCNIDirPrefix string
 }
 
@@ -61,12 +62,16 @@ type cniNetwork struct {
 	CNIConfig     libcni.CNI
 }
 
-func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
+func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
+	if binDir == "" {
+		binDir = DefaultCNIDir
+	}
 	plugin := &cniNetworkPlugin{
 		defaultNetwork:     nil,
-		loNetwork:          getLoNetwork(vendorCNIDirPrefix),
+		loNetwork:          getLoNetwork(binDir, vendorCNIDirPrefix),
 		execer:             utilexec.New(),
 		pluginDir:          pluginDir,
+		binDir:             binDir,
 		vendorCNIDirPrefix: vendorCNIDirPrefix,
 	}
 
@@ -75,11 +80,11 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, vendorCNIDirPrefix str
 	return []network.NetworkPlugin{plugin}
 }
 
-func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
+func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin {
-	return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, "")
+	return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "")
 }
 
-func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, error) {
+func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) {
 	if pluginDir == "" {
 		pluginDir = DefaultNetDir
 	}
@@ -101,7 +106,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er
 		// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
 		vendorDir := vendorCNIDir(vendorCNIDirPrefix, conf.Network.Type)
 		cninet := &libcni.CNIConfig{
-			Path: []string{DefaultCNIDir, vendorDir},
+			Path: []string{binDir, vendorDir},
 		}
 		network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet}
 		return network, nil
@@ -113,7 +118,7 @@ func vendorCNIDir(prefix, pluginType string) string {
 	return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType)
 }
 
-func getLoNetwork(vendorDirPrefix string) *cniNetwork {
+func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
 	loConfig, err := libcni.ConfFromBytes([]byte(`{
   "cniVersion": "0.1.0",
   "name": "cni-loopback",
@@ -125,7 +130,7 @@ func getLoNetwork(vendorDirPrefix string) *cniNetwork {
 		panic(err)
 	}
 	cninet := &libcni.CNIConfig{
-		Path: []string{vendorCNIDir(vendorDirPrefix, loConfig.Network.Type), DefaultCNIDir},
+		Path: []string{vendorCNIDir(vendorDirPrefix, loConfig.Network.Type), binDir},
 	}
 	loNetwork := &cniNetwork{
 		name:          "lo",
@@ -153,7 +158,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentcon
 }
 
 func (plugin *cniNetworkPlugin) syncNetworkConfig() {
-	network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.vendorCNIDirPrefix)
+	network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix)
 	if err != nil {
 		glog.Errorf("error updating cni config: %s", err)
 		return

pkg/kubelet/network/cni/cni_test.go

@@ -183,7 +183,7 @@ func TestCNIPlugin(t *testing.T) {
 		NetnsPath: "/proc/12345/ns/net",
 	}}
 
-	plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), testVendorCNIDirPrefix)
+	plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix)
 	if len(plugins) != 1 {
 		t.Fatalf("Expected only one network plugin, got %d", len(plugins))
 	}
@@ -246,7 +246,7 @@ func TestCNIPlugin(t *testing.T) {
 }
 
 func TestLoNetNonNil(t *testing.T) {
-	if conf := getLoNetwork(""); conf == nil {
+	if conf := getLoNetwork("", ""); conf == nil {
 		t.Error("Expected non-nil lo network")
 	}
 }

pkg/kubelet/network/exec/exec.go

@@ -85,11 +85,16 @@ const (
 	setUpCmd    = "setup"
 	tearDownCmd = "teardown"
 	statusCmd   = "status"
+	defaultDir  = "/usr/libexec/kubernetes/kubelet-plugins/net/exec/"
 )
 
 func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
 	execPlugins := []network.NetworkPlugin{}
 
+	if pluginDir == "" {
+		pluginDir = defaultDir
+	}
+
 	files, _ := ioutil.ReadDir(pluginDir)
 	for _, f := range files {
 		// only directories are counted as plugins