From ea9df6361be0fe9e3f890cb521d9ccde9e467b82 Mon Sep 17 00:00:00 2001 From: wawa0210 Date: Wed, 9 Dec 2020 10:00:08 +0800 Subject: [PATCH] remove windows container hyper-v support --- pkg/features/kube_features.go | 8 ---- pkg/kubelet/apis/BUILD | 14 +----- .../apis/well_known_annotations_windows.go | 46 ------------------- pkg/kubelet/dockershim/BUILD | 1 - pkg/kubelet/dockershim/docker_sandbox.go | 1 - pkg/kubelet/dockershim/helpers_windows.go | 37 +++------------ pkg/kubelet/kuberuntime/BUILD | 3 -- .../kuberuntime_container_windows.go | 28 +++-------- 8 files changed, 16 insertions(+), 122 deletions(-) delete mode 100644 pkg/kubelet/apis/well_known_annotations_windows.go diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 578bca20932..0f51df87b0c 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -183,13 +183,6 @@ const ( // Implement support for limiting pids in pods SupportPodPidsLimit featuregate.Feature = "SupportPodPidsLimit" - // owner: @feiskyer - // alpha: v1.10 - // - // Enable Hyper-V containers on Windows - // Deprecated in 1.20 and removed in 1.21 - HyperVContainer featuregate.Feature = "HyperVContainer" - // owner: @mikedanese // alpha: v1.13 // @@ -709,7 +702,6 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, SupportPodPidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 SupportNodePidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - HyperVContainer: {Default: false, PreRelease: featuregate.Deprecated}, BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, ServiceAccountIssuerDiscovery: {Default: true, PreRelease: featuregate.Beta}, CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/kubelet/apis/BUILD b/pkg/kubelet/apis/BUILD index 823940a76a3..3067d54b40f 100644 --- a/pkg/kubelet/apis/BUILD +++ b/pkg/kubelet/apis/BUILD @@ -7,22 +7,12 @@ load( go_library( name = "go_default_library", - srcs = [ - "well_known_annotations_windows.go", - "well_known_labels.go", - ], + srcs = ["well_known_labels.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/apis", deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/features:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], - "//conditions:default": [], - }), + ], ) filegroup( diff --git a/pkg/kubelet/apis/well_known_annotations_windows.go b/pkg/kubelet/apis/well_known_annotations_windows.go deleted file mode 100644 index 6d577d5f129..00000000000 --- a/pkg/kubelet/apis/well_known_annotations_windows.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apis - -import ( - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" -) - -const ( - // HypervIsolationAnnotationKey is used to run windows containers with hyperv isolation. - // Refer https://aka.ms/hyperv-container. - HypervIsolationAnnotationKey = "experimental.windows.kubernetes.io/isolation-type" - // HypervIsolationValue is used to run windows containers with hyperv isolation. - // Refer https://aka.ms/hyperv-container. - HypervIsolationValue = "hyperv" -) - -// ShouldIsolatedByHyperV returns true if a windows container should be run with hyperv isolation. -func ShouldIsolatedByHyperV(annotations map[string]string) bool { - klog.Warningf("The hyper-v FeatureGate is deprecated in 1.20 and will be removed in 1.21") - - if !utilfeature.DefaultFeatureGate.Enabled(features.HyperVContainer) { - return false - } - - v, ok := annotations[HypervIsolationAnnotationKey] - return ok && v == HypervIsolationValue -} diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 57919f932b8..28d6321188e 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -76,7 +76,6 @@ go_library( "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/winstats:go_default_library", "//vendor/github.com/Microsoft/hcsshim:go_default_library", "//vendor/golang.org/x/sys/windows/registry:go_default_library", diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index 81f8b771897..8408a2ccce3 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -662,7 +662,6 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig, securityOpts := ds.getSandBoxSecurityOpts(securityOptSeparator) hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...) - applyExperimentalCreateConfig(createConfig, c.Annotations) return createConfig, nil } diff --git a/pkg/kubelet/dockershim/helpers_windows.go b/pkg/kubelet/dockershim/helpers_windows.go index e94a0a4bd64..41ef4f4bdd9 100644 --- a/pkg/kubelet/dockershim/helpers_windows.go +++ b/pkg/kubelet/dockershim/helpers_windows.go @@ -29,7 +29,6 @@ import ( "k8s.io/klog/v2" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) // DefaultMemorySwap always returns 0 for no memory swap in a sandbox @@ -50,17 +49,6 @@ func (ds *dockerService) getSandBoxSecurityOpts(separator rune) []string { return nil } -// applyExperimentalCreateConfig applys experimental configures from sandbox annotations. -func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) { - if kubeletapis.ShouldIsolatedByHyperV(annotations) { - createConfig.HostConfig.Isolation = kubeletapis.HypervIsolationValue - - if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" { - createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode("none") - } - } -} - func (ds *dockerService) updateCreateConfig( createConfig *dockertypes.ContainerCreateConfig, config *runtimeapi.ContainerConfig, @@ -68,7 +56,7 @@ func (ds *dockerService) updateCreateConfig( podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error { if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" { createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode) - } else if !kubeletapis.ShouldIsolatedByHyperV(sandboxConfig.Annotations) { + } else { // Todo: Refactor this call in future for calling methods directly in security_context.go modifyHostOptionsForContainer(nil, podSandboxID, createConfig.HostConfig) } @@ -90,8 +78,6 @@ func (ds *dockerService) updateCreateConfig( applyWindowsContainerSecurityContext(wc.GetSecurityContext(), createConfig.Config, createConfig.HostConfig) } - applyExperimentalCreateConfig(createConfig, sandboxConfig.Annotations) - return nil } @@ -149,21 +135,12 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) []string { // Instead of relying on this call, an explicit call to addToNetwork should be // done immediately after ContainerCreation, in case of Windows only. TBD Issue # to handle this - if r.HostConfig.Isolation == kubeletapis.HypervIsolationValue { - // Hyper-V only supports one container per Pod yet and the container will have a different - // IP address from sandbox. Return the first non-sandbox container IP as POD IP. - // TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod. - if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 { - return containerIPs - } - } else { - // Do not return any IP, so that we would continue and get the IP of the Sandbox. - // Windows 1709 and 1803 doesn't have the Namespace support, so getIP() is called - // to replicate the DNS registry key to the Workload container (IP/Gateway/MAC is - // set separately than DNS). - // TODO(feiskyer): remove this workaround after Namespace is supported in Windows RS5. - ds.getIPs(sandboxID, r) - } + // Do not return any IP, so that we would continue and get the IP of the Sandbox. + // Windows 1709 and 1803 doesn't have the Namespace support, so getIP() is called + // to replicate the DNS registry key to the Workload container (IP/Gateway/MAC is + // set separately than DNS). + // TODO(feiskyer): remove this workaround after Namespace is supported in Windows RS5. + ds.getIPs(sandboxID, r) } else { // ds.getIP will call the CNI plugin to fetch the IP if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 { diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 6827346f5d1..c8fcddceb8b 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -87,9 +87,6 @@ go_library( "//pkg/kubelet/qos:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", ], - "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/kubelet/apis:go_default_library", - ], "//conditions:default": [], }), ) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go index 49fec206c4a..172d2ded604 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go @@ -25,7 +25,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" kubefeatures "k8s.io/kubernetes/pkg/features" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/securitycontext" @@ -52,7 +51,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 } cpuLimit := container.Resources.Limits.Cpu() - isolatedByHyperv := kubeletapis.ShouldIsolatedByHyperV(pod.Annotations) if !cpuLimit.IsZero() { // Note that sysinfo.NumCPU() is limited to 64 CPUs on Windows due to Processor Groups, // as only 64 processors are available for execution by a given process. This causes @@ -85,16 +83,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 cpuMaximum := 10000 * cpuLimit.MilliValue() / int64(runtime.NumCPU()) / 1000 - // TODO: This should be reviewed or removed once Hyper-V support is implemented with CRI-ContainerD - // in a future release. cpuCount may or may not be required if cpuMaximum is set. - if isolatedByHyperv { - cpuCount := int64(cpuLimit.MilliValue()+999) / 1000 - wc.Resources.CpuCount = cpuCount - - if cpuCount != 0 { - cpuMaximum = cpuLimit.MilliValue() / cpuCount * 10000 / 1000 - } - } // ensure cpuMaximum is in range [1, 10000]. if cpuMaximum < 1 { cpuMaximum = 1 @@ -105,15 +93,13 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 wc.Resources.CpuMaximum = cpuMaximum } - if !isolatedByHyperv { - // The processor resource controls are mutually exclusive on - // Windows Server Containers, the order of precedence is - // CPUCount first, then CPUMaximum. - if wc.Resources.CpuCount > 0 { - if wc.Resources.CpuMaximum > 0 { - wc.Resources.CpuMaximum = 0 - klog.Warningf("Mutually exclusive options: CPUCount priority > CPUMaximum priority on Windows Server Containers. CPUMaximum should be ignored") - } + // The processor resource controls are mutually exclusive on + // Windows Server Containers, the order of precedence is + // CPUCount first, then CPUMaximum. + if wc.Resources.CpuCount > 0 { + if wc.Resources.CpuMaximum > 0 { + wc.Resources.CpuMaximum = 0 + klog.Warningf("Mutually exclusive options: CPUCount priority > CPUMaximum priority on Windows Server Containers. CPUMaximum should be ignored") } }