Merge pull request #64445 from squeed/more-cni-capabilities

Automatic merge from submit-queue (batch tested with PRs 64445, 67459, 67434). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

dockershim/network: pass ipRange CNI capabilities

**What this PR does / why we need it**:
Updates the dynamic (capability args) passed from Kubernetes to the CNI plugin. This means CNI plugin authors can offer more features and / or reduce their dependency on the APIServer.

Currently, we only pass the `portMappings` capability. CNI now supports `bandwidth` for bandwidth limiting and `ipRanges` for preferred IP blocks. This PR adds support for these two new capabilities.

Bandwidth limits are provided - as implemented in kubenet - via the pod annotations `kubernetes.io/ingress-bandwidth` and `kubernetes.io/egress-bandwidth`.

The ipRanges field simply passes the PodCIDR. This does mean that we need to change the NodeReady algorithm. Previously, we would only set NodeNotReady on missing PodCIDR when using Kubenet. Now, if the CNI configuration includes the `ipRanges` capability, we need to do the same.

**Which issue(s) this PR fixes**:
Fixes #64393

**Release note**:

```release-note
The dockershim now sets the "bandwidth" and "ipRanges" CNI capabilities (dynamic parameters). Plugin authors and administrators can now take advantage of this by updating their CNI configuration file. For more information, see the [CNI docs](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration)
```

pkg/kubelet/dockershim/network/cni/cni.go

@@ -50,6 +50,7 @@ type cniNetworkPlugin struct {
 	nsenterPath string
 	confDir     string
 	binDirs     []string
+	podCidr     string
 }
 
 type cniNetwork struct {
@@ -83,6 +84,11 @@ type cniBandwidthEntry struct {
 	EgressBurst int `json:"egressBurst,omitempty"`
 }
 
+// cniIpRange maps to the standard CNI ip range Capability
+type cniIpRange struct {
+	Subnet string `json:"subnet"`
+}
+
 func SplitDirs(dirs string) []string {
 	// Use comma rather than colon to work better with Windows too
 	return strings.Split(dirs, ",")
@@ -152,6 +158,8 @@ func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error)
 			continue
 		}
 
+		glog.V(4).Infof("Using CNI configuration file %s", confFile)
+
 		network := &cniNetwork{
 			name:          confList.Name,
 			NetworkConfig: confList,
@@ -199,9 +207,43 @@ func (plugin *cniNetworkPlugin) checkInitialized() error {
 	if plugin.getDefaultNetwork() == nil {
 		return errors.New("cni config uninitialized")
 	}
+
+	// If the CNI configuration has the ipRanges capability, we need a PodCIDR assigned
+	for _, p := range plugin.getDefaultNetwork().NetworkConfig.Plugins {
+		if p.Network.Capabilities["ipRanges"] {
+			if plugin.podCidr == "" {
+				return errors.New("no PodCIDR set")
+			}
+			break
+		}
+	}
 	return nil
 }
 
+// Event handles any change events. The only event ever sent is the PodCIDR change.
+// No network plugins support changing an already-set PodCIDR
+func (plugin *cniNetworkPlugin) Event(name string, details map[string]interface{}) {
+	if name != network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE {
+		return
+	}
+
+	plugin.Lock()
+	defer plugin.Unlock()
+
+	podCIDR, ok := details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR].(string)
+	if !ok {
+		glog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE)
+		return
+	}
+
+	if plugin.podCidr != "" {
+		glog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR)
+		return
+	}
+
+	plugin.podCidr = podCIDR
+}
+
 func (plugin *cniNetworkPlugin) Name() string {
 	return CNIPluginName
 }
@@ -346,5 +388,8 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string
 		rt.CapabilityArgs["bandwidth"] = bandwidthParam
 	}
 
+	// Set the PodCIDR
+	rt.CapabilityArgs["ipRanges"] = [][]cniIpRange{{{Subnet: plugin.podCidr}}}
+
 	return rt, nil
 }

pkg/kubelet/dockershim/network/cni/cni_test.go

@@ -61,7 +61,7 @@ func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir,
 	if err != nil {
 		t.Fatalf("Failed to install plugin %s: %v", confFile, err)
 	}
-	networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true, "bandwidth": true}  }`, confName, binName)
+	networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true, "bandwidth": true, "ipRanges": true}  }`, confName, binName)
 	_, err = f.WriteString(networkConfig)
 	if err != nil {
 		t.Fatalf("Failed to write network config file (%v)", err)
@@ -218,6 +218,19 @@ func TestCNIPlugin(t *testing.T) {
 
 	mockLoCNI.On("AddNetworkList", cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&types020.Result{IP4: &types020.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil)
 
+	// Check that status returns an error
+	if err := cniPlugin.Status(); err == nil {
+		t.Fatalf("cniPlugin returned non-err with no podCidr")
+	}
+
+	cniPlugin.Event(network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE, map[string]interface{}{
+		network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR: "10.0.2.0/24",
+	})
+
+	if err := cniPlugin.Status(); err != nil {
+		t.Fatalf("unexpected status err: %v", err)
+	}
+
 	ports := map[string][]*hostport.PortMapping{
 		containerID.ID: {
 			{
@@ -259,8 +272,9 @@ func TestCNIPlugin(t *testing.T) {
 	// Verify the correct network configuration was passed
 	inputConfig := struct {
 		RuntimeConfig struct {
-			Bandwidth    map[string]interface{}   `json:"bandwidth"`
 			PortMappings []map[string]interface{}   `json:"portMappings"`
+			Bandwidth    map[string]interface{}     `json:"bandwidth"`
+			IpRanges     [][]map[string]interface{} `json:"ipRanges"`
 		} `json:"runtimeConfig"`
 	}{}
 	inputBytes, inerr := ioutil.ReadFile(inputFile)
@@ -282,6 +296,16 @@ func TestCNIPlugin(t *testing.T) {
 		t.Errorf("mismatch in expected bandwidth. expected %v got %v", expectedBandwidth, inputConfig.RuntimeConfig.Bandwidth)
 	}
 
+	expectedIpRange := [][]map[string]interface{}{
+		{
+			{"subnet": "10.0.2.0/24"},
+		},
+	}
+
+	if !reflect.DeepEqual(inputConfig.RuntimeConfig.IpRanges, expectedIpRange) {
+		t.Errorf("mismatch in expected ipRange. expected %v got %v", expectedIpRange, inputConfig.RuntimeConfig.IpRanges)
+	}
+
 	// Get its IP address
 	status, err := plug.GetPodNetworkStatus("podNamespace", "podName", containerID)
 	if err != nil {