From bd05e1af2b754ceca7203d9a930939dbdfddbfe1 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Fri, 3 Feb 2017 13:11:01 -0800 Subject: [PATCH] add portmapping getter into network host --- pkg/kubelet/dockershim/docker_service.go | 73 ++++++++++++++++++++---- pkg/kubelet/kubelet.go | 2 +- pkg/kubelet/network/cni/cni_test.go | 2 + pkg/kubelet/network/plugins.go | 31 +++++++--- pkg/kubelet/network/testing/fake_host.go | 10 ++++ pkg/kubelet/networks.go | 3 + 6 files changed, 101 insertions(+), 20 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index f206d6474cd..aa8601bd759 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -25,6 +25,7 @@ import ( dockertypes "github.com/docker/engine-api/types" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/componentconfig" internalapi "k8s.io/kubernetes/pkg/kubelet/api" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -34,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network/cni" + "k8s.io/kubernetes/pkg/kubelet/network/hostport" "k8s.io/kubernetes/pkg/kubelet/network/kubenet" "k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/util/cache" @@ -107,6 +109,35 @@ type NetworkPluginSettings struct { LegacyRuntimeHost network.LegacyHost } +// namespaceGetter is a wrapper around the dockerService that implements +// the network.NamespaceGetter interface. +type namespaceGetter struct { + ds *dockerService +} + +func (n *namespaceGetter) GetNetNS(containerID string) (string, error) { + return n.ds.GetNetNS(containerID) +} + +// portMappingGetter is a wrapper around the dockerService that implements +// the network.PortMappingGetter interface. +type portMappingGetter struct { + ds *dockerService +} + +func (p *portMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) { + return p.ds.GetPodPortMappings(containerID) +} + +// dockerNetworkHost implements network.Host by wrapping the legacy host passed in by the kubelet +// and dockerServices which implementes the rest of the network host interfaces. +// The legacy host methods are slated for deletion. +type dockerNetworkHost struct { + network.LegacyHost + *namespaceGetter + *portMappingGetter +} + var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPathLabelKey, sandboxIDLabelKey} // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process. @@ -138,6 +169,7 @@ func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot str netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, &namespaceGetter{ds}, + &portMappingGetter{ds}, } plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU) if err != nil { @@ -240,12 +272,6 @@ func (ds *dockerService) UpdateRuntimeConfig(runtimeConfig *runtimeapi.RuntimeCo return } -// namespaceGetter is a wrapper around the dockerService that implements -// the network.NamespaceGetter interface. -type namespaceGetter struct { - *dockerService -} - // GetNetNS returns the network namespace of the given containerID. The ID // supplied is typically the ID of a pod sandbox. This getter doesn't try // to map non-sandbox IDs to their respective sandboxes. @@ -257,12 +283,24 @@ func (ds *dockerService) GetNetNS(podSandboxID string) (string, error) { return getNetworkNamespace(r), nil } -// dockerNetworkHost implements network.Host by wrapping the legacy host -// passed in by the kubelet and adding NamespaceGetter methods. The legacy -// host methods are slated for deletion. -type dockerNetworkHost struct { - network.LegacyHost - *namespaceGetter +// GetPodPortMappings returns the port mappings of the given podSandbox ID. +func (ds *dockerService) GetPodPortMappings(podSandboxID string) ([]*hostport.PortMapping, error) { + // TODO: get portmappings from docker labels for backward compatibility + checkpoint, err := ds.checkpointHandler.GetCheckpoint(podSandboxID) + if err != nil { + return nil, err + } + + portMappings := []*hostport.PortMapping{} + for _, pm := range checkpoint.Data.PortMappings { + proto := toAPIProtocol(*pm.Protocol) + portMappings = append(portMappings, &hostport.PortMapping{ + HostPort: *pm.HostPort, + ContainerPort: *pm.ContainerPort, + Protocol: proto, + }) + } + return portMappings, nil } // Start initializes and starts components in dockerService. @@ -351,3 +389,14 @@ func (ds *dockerService) getDockerVersionFromCache() (*dockertypes.Version, erro } return dv, nil } + +func toAPIProtocol(protocol Protocol) v1.Protocol { + switch protocol { + case protocolTCP: + return v1.ProtocolTCP + case protocolUDP: + return v1.ProtocolUDP + } + glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol) + return v1.ProtocolTCP +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 65e4bdfc2cf..c2f38a54894 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -484,7 +484,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub } glog.Infof("Hairpin mode set to %q", klet.hairpinMode) - if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &criNetworkHost{&networkHost{klet}}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil { + if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &criNetworkHost{&networkHost{klet}, &network.NoopPortMappingGetter{}}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil { return nil, err } else { klet.networkPlugin = plug diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index c2c8a1e73de..1d76236fea8 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -39,6 +39,7 @@ import ( containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network/cni/testing" + networktest "k8s.io/kubernetes/pkg/kubelet/network/testing" utilexec "k8s.io/kubernetes/pkg/util/exec" ) @@ -111,6 +112,7 @@ func tearDownPlugin(tmpDir string) { } type fakeNetworkHost struct { + networktest.FakePortMappingGetter kubeClient clientset.Interface runtime kubecontainer.Runtime } diff --git a/pkg/kubelet/network/plugins.go b/pkg/kubelet/network/plugins.go index ec59e09d8cc..ed925562e1c 100644 --- a/pkg/kubelet/network/plugins.go +++ b/pkg/kubelet/network/plugins.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/network/hostport" utilexec "k8s.io/kubernetes/pkg/util/exec" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" ) @@ -111,9 +112,9 @@ type LegacyHost interface { // Only used for hostport management GetRuntime() kubecontainer.Runtime - // SupportsLegacyFeaturs returns true if this host can support hostports - // and bandwidth shaping. Both will either get added to CNI or dropped, - // so differnt implementations can choose to ignore them. + // SupportsLegacyFeatures returns true if the network host support GetPodByName, KubeClient interface and kubelet + // runtime interface. These interfaces will no longer be implemented by CRI shims. + // This function helps network plugins to choose their behavior based on runtime. SupportsLegacyFeatures() bool } @@ -121,17 +122,19 @@ type LegacyHost interface { // TODO(#35457): get rid of this backchannel to the kubelet. The scope of // the back channel is restricted to host-ports/testing, and restricted // to kubenet. No other network plugin wrapper needs it. Other plugins -// only require a way to access namespace information, which they can do -// directly through the embedded NamespaceGetter. +// only require a way to access namespace information and port mapping +// information , which they can do directly through the embedded interfaces. type Host interface { // NamespaceGetter is a getter for sandbox namespace information. - // It's the only part of this interface that isn't currently deprecated. NamespaceGetter + // PortMappingGetter is a getter for sandbox port mapping information. + PortMappingGetter + // LegacyHost contains methods that trap back into the Kubelet. Dependence // *do not* add more dependencies in this interface. In a post-cri world, // network plugins will be invoked by the runtime shim, and should only - // require NamespaceGetter. + // require GetNetNS and GetPodPortMappings. LegacyHost } @@ -143,6 +146,14 @@ type NamespaceGetter interface { GetNetNS(containerID string) (string, error) } +// PortMappingGetter is an interface to retrieve port mapping information for a given +// sandboxID. Typically implemented by runtime shims that are closely coupled to +// CNI plugin wrappers like kubenet. +type PortMappingGetter interface { + // GetPodPortMappings returns sandbox port mappings information. + GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) +} + // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names. func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) (NetworkPlugin, error) { if networkPluginName == "" { @@ -276,3 +287,9 @@ func GetPodIP(execer utilexec.Interface, nsenterPath, netnsPath, interfaceName s return ip, nil } + +type NoopPortMappingGetter struct{} + +func (*NoopPortMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) { + return nil, nil +} diff --git a/pkg/kubelet/network/testing/fake_host.go b/pkg/kubelet/network/testing/fake_host.go index 61761a8c542..ba1922f282a 100644 --- a/pkg/kubelet/network/testing/fake_host.go +++ b/pkg/kubelet/network/testing/fake_host.go @@ -24,10 +24,12 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" + "k8s.io/kubernetes/pkg/kubelet/network/hostport" ) type fakeNetworkHost struct { fakeNamespaceGetter + FakePortMappingGetter kubeClient clientset.Interface Legacy bool Runtime *containertest.FakeRuntime @@ -61,3 +63,11 @@ type fakeNamespaceGetter struct { func (nh *fakeNamespaceGetter) GetNetNS(containerID string) (string, error) { return nh.ns, nil } + +type FakePortMappingGetter struct { + mem map[string][]*hostport.PortMapping +} + +func (pm *FakePortMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) { + return pm.mem[containerID], nil +} diff --git a/pkg/kubelet/networks.go b/pkg/kubelet/networks.go index 6481fe87209..d8382433372 100644 --- a/pkg/kubelet/networks.go +++ b/pkg/kubelet/networks.go @@ -20,6 +20,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/network" ) // This just exports required functions from kubelet proper, for use by network @@ -54,6 +55,8 @@ func (nh *networkHost) SupportsLegacyFeatures() bool { // methods, because networkHost is slated for deletion. type criNetworkHost struct { *networkHost + // criNetworkHost currently support legacy features. Hence no need to support PortMappingGetter + *network.NoopPortMappingGetter } // GetNetNS returns the network namespace of the given containerID.