From d7f9e22815aede0b1ab77e0b8f87331e4f214e2a Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Wed, 14 Jun 2017 16:27:39 -0700 Subject: [PATCH] Sandbox Support for Windows + CNI Following are part of this commit +++++++++++++++++++++++++++++++++ * Windows CNI Support (1) Support to use --network-plugin=cni (2) Handled platform requirement of calling CNI ADD for all the containers. (2.1) For POD Infra container, netNs has to be empty (2.2) For all other containers, sharing the network namespace of POD container, should pass netNS name as "container:", same as the NetworkMode of the current container (2.3) The Windows CNI plugin has to handle this to call into Platform. Sample Windows CNI Plugin code to be shared soon. * Sandbox support for Windows (1) Sandbox support for Windows. Works only with Docker runtime. (2) Retained CONTAINER_NETWORK as a backward compatibilty flag, to not break existing deployments using it. (3) Works only with CNI plugin enabled. (*) Changes to reinvoke CNI ADD for every new container created. This is hooked up with PodStatus, but would be ideal to move it outside of this, once we have CNI GET support --- pkg/kubelet/dockershim/helpers.go | 8 -- pkg/kubelet/dockershim/helpers_linux.go | 8 ++ pkg/kubelet/dockershim/helpers_unsupported.go | 6 ++ pkg/kubelet/dockershim/helpers_windows.go | 42 +++++++++- pkg/kubelet/network/cni/BUILD | 17 +++- pkg/kubelet/network/cni/cni.go | 58 ++------------ pkg/kubelet/network/cni/cni_others.go | 80 +++++++++++++++++++ pkg/kubelet/network/cni/cni_windows.go | 61 ++++++++++++++ 8 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 pkg/kubelet/network/cni/cni_others.go create mode 100644 pkg/kubelet/network/cni/cni_windows.go diff --git a/pkg/kubelet/dockershim/helpers.go b/pkg/kubelet/dockershim/helpers.go index c6f442c78af..595263a840b 100644 --- a/pkg/kubelet/dockershim/helpers.go +++ b/pkg/kubelet/dockershim/helpers.go @@ -223,14 +223,6 @@ func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separ return fmtOpts, nil } -func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { - if c.State.Pid == 0 { - // Docker reports pid 0 for an exited container. - return "", fmt.Errorf("Cannot find network namespace for the terminated container %q", c.ID) - } - return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil -} - // dockerFilter wraps around dockerfilters.Args and provides methods to modify // the filter easily. type dockerFilter struct { diff --git a/pkg/kubelet/dockershim/helpers_linux.go b/pkg/kubelet/dockershim/helpers_linux.go index 18a223d755d..a53ecce1777 100644 --- a/pkg/kubelet/dockershim/helpers_linux.go +++ b/pkg/kubelet/dockershim/helpers_linux.go @@ -133,3 +133,11 @@ func (ds *dockerService) updateCreateConfig( func (ds *dockerService) determinePodIPBySandboxID(uid string) string { return "" } + +func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { + if c.State.Pid == 0 { + // Docker reports pid 0 for an exited container. + return "", fmt.Errorf("cannot find network namespace for the terminated container %q", c.ID) + } + return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil +} diff --git a/pkg/kubelet/dockershim/helpers_unsupported.go b/pkg/kubelet/dockershim/helpers_unsupported.go index db24ae7afb6..dfe3a097a49 100644 --- a/pkg/kubelet/dockershim/helpers_unsupported.go +++ b/pkg/kubelet/dockershim/helpers_unsupported.go @@ -19,6 +19,8 @@ limitations under the License. package dockershim import ( + "fmt" + "github.com/blang/semver" dockertypes "github.com/docker/docker/api/types" "github.com/golang/glog" @@ -47,3 +49,7 @@ func (ds *dockerService) determinePodIPBySandboxID(uid string) string { glog.Warningf("determinePodIPBySandboxID is unsupported in this build") return "" } + +func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { + return "", fmt.Errorf("unsupported platform") +} diff --git a/pkg/kubelet/dockershim/helpers_windows.go b/pkg/kubelet/dockershim/helpers_windows.go index b9ddfd2c649..c93515daab2 100644 --- a/pkg/kubelet/dockershim/helpers_windows.go +++ b/pkg/kubelet/dockershim/helpers_windows.go @@ -47,6 +47,9 @@ func (ds *dockerService) updateCreateConfig( podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error { if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" { createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode) + } else { + // Todo: Refactor this call in future for calling methods directly in security_context.go + modifyHostNetworkOptionForContainer(false, podSandboxID, createConfig.HostConfig) } return nil @@ -71,14 +74,49 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string { if err != nil { continue } - if containerIP := getContainerIP(r); containerIP != "" { - return containerIP + + // Versions and feature support + // ============================ + // Windows version == Windows Server, Version 1709,, Supports both sandbox and non-sandbox case + // Windows version == Windows Server 2016 Support only non-sandbox case + // Windows version < Windows Server 2016 is Not Supported + + // Sandbox support in Windows mandates CNI Plugin. + // Presense of CONTAINER_NETWORK flag is considered as non-Sandbox cases here + + // Todo: Add a kernel version check for more validation + + if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" { + // Do not return any IP, so that we would continue and get the IP of the Sandbox + ds.getIP(sandboxID, r) + } else { + // On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network, + // with the shared container name as NetNS info, + // This is passed down to the platform to replicate some necessary information to the new container + + // + // This place is chosen as a hack for now, since getContainerIP would end up calling CNI's addToNetwork + // That is why addToNetwork is required to be idempotent + + // Instead of relying on this call, an explicit call to addToNetwork should be + // done immediately after ContainerCreation, in case of Windows only. TBD Issue # to handle this + + if containerIP := getContainerIP(r); containerIP != "" { + return containerIP + } } } return "" } +func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { + // Currently in windows there is no identifier exposed for network namespace + // Like docker, the referenced container id is used to figure out the network namespace id internally by the platform + // so returning the docker networkMode (which holds container: for network namespace here + return string(c.HostConfig.NetworkMode), nil +} + func getContainerIP(container *dockertypes.ContainerJSON) string { if container.NetworkSettings != nil { for _, network := range container.NetworkSettings.Networks { diff --git a/pkg/kubelet/network/cni/BUILD b/pkg/kubelet/network/cni/BUILD index 0ccb25e75e4..b91992366ec 100644 --- a/pkg/kubelet/network/cni/BUILD +++ b/pkg/kubelet/network/cni/BUILD @@ -8,7 +8,15 @@ load( go_library( name = "go_default_library", - srcs = ["cni.go"], + srcs = [ + "cni.go", + "cni_others.go", + ] + select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "cni_windows.go", + ], + "//conditions:default": [], + }), importpath = "k8s.io/kubernetes/pkg/kubelet/network/cni", deps = [ "//pkg/kubelet/apis/kubeletconfig:go_default_library", @@ -18,7 +26,12 @@ go_library( "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 0ae1edc801d..5649cc1352e 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -153,37 +153,12 @@ func vendorCNIDir(prefix, pluginType string) string { return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType) } -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { - loConfig, err := libcni.ConfListFromBytes([]byte(`{ - "cniVersion": "0.2.0", - "name": "cni-loopback", - "plugins":[{ - "type": "loopback" - }] -}`)) - if err != nil { - // The hardcoded config above should always be valid and unit tests will - // catch this - panic(err) - } - cninet := &libcni.CNIConfig{ - Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir}, - } - loNetwork := &cniNetwork{ - name: "lo", - NetworkConfig: loConfig, - CNIConfig: cninet, - } - - return loNetwork -} - func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error { - var err error - plugin.nsenterPath, err = plugin.execer.LookPath("nsenter") + err := plugin.platformInit() if err != nil { return err } + plugin.host = host plugin.syncNetworkConfig() @@ -239,10 +214,12 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } - _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath) - if err != nil { - glog.Errorf("Error while adding to cni lo network: %s", err) - return err + // Windows doesn't have loNetwork. It comes only with Linux + if plugin.loNetwork != nil { + if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath); err != nil { + glog.Errorf("Error while adding to cni lo network: %s", err) + return err + } } _, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath) @@ -268,25 +245,6 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath) } -// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin. -// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls -func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) { - netnsPath, err := plugin.host.GetNetNS(id.ID) - if err != nil { - return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) - } - if netnsPath == "" { - return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id) - } - - ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName) - if err != nil { - return nil, err - } - - return &network.PodNetworkStatus{IP: ip}, nil -} - func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) { rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath) if err != nil { diff --git a/pkg/kubelet/network/cni/cni_others.go b/pkg/kubelet/network/cni/cni_others.go new file mode 100644 index 00000000000..735db9cc69c --- /dev/null +++ b/pkg/kubelet/network/cni/cni_others.go @@ -0,0 +1,80 @@ +// +build !windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cni + +import ( + "fmt" + + "github.com/containernetworking/cni/libcni" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/network" +) + +func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { + loConfig, err := libcni.ConfListFromBytes([]byte(`{ + "cniVersion": "0.2.0", + "name": "cni-loopback", + "plugins":[{ + "type": "loopback" + }] +}`)) + if err != nil { + // The hardcoded config above should always be valid and unit tests will + // catch this + panic(err) + } + cninet := &libcni.CNIConfig{ + Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir}, + } + loNetwork := &cniNetwork{ + name: "lo", + NetworkConfig: loConfig, + CNIConfig: cninet, + } + + return loNetwork +} + +func (plugin *cniNetworkPlugin) platformInit() error { + var err error + plugin.nsenterPath, err = plugin.execer.LookPath("nsenter") + if err != nil { + return err + } + return nil +} + +// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin. +// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls +func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) { + netnsPath, err := plugin.host.GetNetNS(id.ID) + if err != nil { + return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) + } + if netnsPath == "" { + return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id) + } + + ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName) + if err != nil { + return nil, err + } + + return &network.PodNetworkStatus{IP: ip}, nil +} diff --git a/pkg/kubelet/network/cni/cni_windows.go b/pkg/kubelet/network/cni/cni_windows.go new file mode 100644 index 00000000000..9a0b17c8f11 --- /dev/null +++ b/pkg/kubelet/network/cni/cni_windows.go @@ -0,0 +1,61 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cni + +import ( + "fmt" + + cniTypes020 "github.com/containernetworking/cni/pkg/types/020" + "github.com/golang/glog" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/network" +) + +func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { + return nil +} + +func (plugin *cniNetworkPlugin) platformInit() error { + return nil +} + +// GetPodNetworkStatus : Assuming addToNetwork is idempotent, we can call this API as many times as required to get the IPAddress +func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) { + netnsPath, err := plugin.host.GetNetNS(id.ID) + if err != nil { + return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) + } + + result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath) + + glog.V(5).Infof("GetPodNetworkStatus result %+v", result) + if err != nil { + glog.Errorf("error while adding to cni network: %s", err) + return nil, err + } + + // Parse the result and get the IPAddress + var result020 *cniTypes020.Result + result020, err = cniTypes020.GetResult(result) + if err != nil { + glog.Errorf("error while cni parsing result: %s", err) + return nil, err + } + return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil +}