mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #63194 from m1093782566/cni-ts
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Adding traffic shaping support for CNI network driver **What this PR does / why we need it**: Adding traffic shaping support for CNI network driver - it's also a sub-task of kubenet deprecation work. Design document is available here: https://github.com/kubernetes/community/pull/1893 **Which issue(s) this PR fixes**: Fixes # **Special notes for your reviewer**: /cc @freehan @jingax10 @caseydavenport @dcbw /sig network /sig node **Release note**: ```release-note Support traffic shaping for CNI network driver ```
This commit is contained in:
commit
e943d09fa3
@ -18,6 +18,7 @@ go_library(
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/container:go_default_library",
|
||||
"//pkg/kubelet/dockershim/network:go_default_library",
|
||||
"//pkg/util/bandwidth:go_default_library",
|
||||
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
|
||||
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
|
@ -29,6 +29,7 @@ import (
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/pkg/kubelet/dockershim/network"
|
||||
"k8s.io/kubernetes/pkg/util/bandwidth"
|
||||
utilexec "k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
@ -66,6 +67,22 @@ type cniPortMapping struct {
|
||||
HostIP string `json:"hostIP"`
|
||||
}
|
||||
|
||||
// cniBandwidthEntry maps to the standard CNI bandwidth Capability
|
||||
// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md and
|
||||
// https://github.com/containernetworking/plugins/blob/master/plugins/meta/bandwidth/README.md
|
||||
type cniBandwidthEntry struct {
|
||||
// IngressRate is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
|
||||
IngressRate int `json:"ingressRate,omitempty"`
|
||||
// IngressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
|
||||
// NOTE: it's not used for now and default to 0.
|
||||
IngressBurst int `json:"ingressBurst,omitempty"`
|
||||
// EgressRate is the bandwidth is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
|
||||
EgressRate int `json:"egressRate,omitempty"`
|
||||
// EgressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
|
||||
// NOTE: it's not used for now and default to 0.
|
||||
EgressBurst int `json:"egressBurst,omitempty"`
|
||||
}
|
||||
|
||||
func SplitDirs(dirs string) []string {
|
||||
// Use comma rather than colon to work better with Windows too
|
||||
return strings.Split(dirs, ",")
|
||||
@ -208,13 +225,13 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec
|
||||
|
||||
// Windows doesn't have loNetwork. It comes only with Linux
|
||||
if plugin.loNetwork != nil {
|
||||
if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath); err != nil {
|
||||
if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations); err != nil {
|
||||
glog.Errorf("Error while adding to cni lo network: %s", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
|
||||
_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations)
|
||||
if err != nil {
|
||||
glog.Errorf("Error while adding to cni network: %s", err)
|
||||
return err
|
||||
@ -234,11 +251,11 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
|
||||
glog.Warningf("CNI failed to retrieve network namespace path: %v", err)
|
||||
}
|
||||
|
||||
return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
|
||||
return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil)
|
||||
}
|
||||
|
||||
func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) {
|
||||
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
|
||||
func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (cnitypes.Result, error) {
|
||||
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations)
|
||||
if err != nil {
|
||||
glog.Errorf("Error adding network when building cni runtime conf: %v", err)
|
||||
return nil, err
|
||||
@ -255,8 +272,8 @@ func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) error {
|
||||
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
|
||||
func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) error {
|
||||
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations)
|
||||
if err != nil {
|
||||
glog.Errorf("Error deleting network when building cni runtime conf: %v", err)
|
||||
return err
|
||||
@ -274,7 +291,7 @@ func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (*libcni.RuntimeConf, error) {
|
||||
func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (*libcni.RuntimeConf, error) {
|
||||
glog.V(4).Infof("Got netns path %v", podNetnsPath)
|
||||
glog.V(4).Infof("Using podns path %v", podNs)
|
||||
|
||||
@ -312,5 +329,22 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string
|
||||
"portMappings": portMappingsParam,
|
||||
}
|
||||
|
||||
ingress, egress, err := bandwidth.ExtractPodBandwidthResources(annotations)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error reading pod bandwidth annotations: %v", err)
|
||||
}
|
||||
if ingress != nil || egress != nil {
|
||||
bandwidthParam := cniBandwidthEntry{}
|
||||
if ingress != nil {
|
||||
bandwidthParam.IngressRate = int(ingress.Value() / 1000)
|
||||
bandwidthParam.IngressBurst = 0 // default to no limit
|
||||
}
|
||||
if egress != nil {
|
||||
bandwidthParam.EgressRate = int(egress.Value() / 1000)
|
||||
bandwidthParam.EgressBurst = 0 // default to no limit
|
||||
}
|
||||
rt.CapabilityArgs["bandwidth"] = bandwidthParam
|
||||
}
|
||||
|
||||
return rt, nil
|
||||
}
|
||||
|
@ -61,7 +61,7 @@ func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir,
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to install plugin %s: %v", confFile, err)
|
||||
}
|
||||
networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, confName, binName)
|
||||
networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true, "bandwidth": true} }`, confName, binName)
|
||||
_, err = f.WriteString(networkConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to write network config file (%v)", err)
|
||||
@ -236,8 +236,12 @@ func TestCNIPlugin(t *testing.T) {
|
||||
t.Fatalf("Failed to select the desired plugin: %v", err)
|
||||
}
|
||||
|
||||
bandwidthAnnotation := make(map[string]string)
|
||||
bandwidthAnnotation["kubernetes.io/ingress-bandwidth"] = "1M"
|
||||
bandwidthAnnotation["kubernetes.io/egress-bandwidth"] = "1M"
|
||||
|
||||
// Set up the pod
|
||||
err = plug.SetUpPod("podNamespace", "podName", containerID, map[string]string{})
|
||||
err = plug.SetUpPod("podNamespace", "podName", containerID, bandwidthAnnotation)
|
||||
if err != nil {
|
||||
t.Errorf("Expected nil: %v", err)
|
||||
}
|
||||
@ -255,6 +259,7 @@ func TestCNIPlugin(t *testing.T) {
|
||||
// Verify the correct network configuration was passed
|
||||
inputConfig := struct {
|
||||
RuntimeConfig struct {
|
||||
Bandwidth map[string]interface{} `json:"bandwidth"`
|
||||
PortMappings []map[string]interface{} `json:"portMappings"`
|
||||
} `json:"runtimeConfig"`
|
||||
}{}
|
||||
@ -270,6 +275,12 @@ func TestCNIPlugin(t *testing.T) {
|
||||
if !reflect.DeepEqual(inputConfig.RuntimeConfig.PortMappings, expectedMappings) {
|
||||
t.Errorf("mismatch in expected port mappings. expected %v got %v", expectedMappings, inputConfig.RuntimeConfig.PortMappings)
|
||||
}
|
||||
expectedBandwidth := map[string]interface{}{
|
||||
"ingressRate": 1000.0, "egressRate": 1000.0,
|
||||
}
|
||||
if !reflect.DeepEqual(inputConfig.RuntimeConfig.Bandwidth, expectedBandwidth) {
|
||||
t.Errorf("mismatch in expected bandwidth. expected %v got %v", expectedBandwidth, inputConfig.RuntimeConfig.Bandwidth)
|
||||
}
|
||||
|
||||
// Get its IP address
|
||||
status, err := plug.GetPodNetworkStatus("podNamespace", "podName", containerID)
|
||||
|
@ -42,7 +42,7 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin
|
||||
return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
|
||||
}
|
||||
|
||||
result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
|
||||
result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil)
|
||||
|
||||
glog.V(5).Infof("GetPodNetworkStatus result %+v", result)
|
||||
if err != nil {
|
||||
|
Loading…
Reference in New Issue
Block a user