diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 3ae5ab8f044..0c22927e040 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -115,70 +115,60 @@ go_library( "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ - "//pkg/features:go_default_library", "//pkg/proxy/metrics:go_default_library", "//pkg/util/dbus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", @@ -236,7 +226,6 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", - "//pkg/features:go_default_library", "//pkg/proxy/apis/kubeproxyconfig:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/pointer:go_default_library", @@ -247,52 +236,42 @@ go_test( "@io_bazel_rules_go//go/platform:android": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ "//pkg/proxy/ipvs:go_default_library", "//pkg/util/iptables:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 1ab70b1ab52..7c87a59551d 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -30,9 +30,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" proxyconfig "k8s.io/kubernetes/pkg/proxy/config" @@ -276,21 +274,13 @@ func newProxyServer( } func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string { - if proxyMode == proxyModeUserspace { + switch proxyMode { + case proxyModeUserspace: return proxyModeUserspace - } - - if len(proxyMode) > 0 && proxyMode == proxyModeIPTables { + case proxyModeIPTables: return tryIPTablesProxy(iptver, kcompat) - } - - if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) { - if proxyMode == proxyModeIPVS { - return tryIPVSProxy(iptver, khandle, ipsetver, kcompat) - } else { - glog.Warningf("Can't use ipvs proxier, trying iptables proxier") - return tryIPTablesProxy(iptver, kcompat) - } + case proxyModeIPVS: + return tryIPVSProxy(iptver, khandle, ipsetver, kcompat) } glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) return tryIPTablesProxy(iptver, kcompat) diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index 01cc00dc18b..f439f6186f9 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -22,7 +22,6 @@ import ( "fmt" "testing" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/proxy/ipvs" "k8s.io/kubernetes/pkg/util/iptables" ) @@ -88,19 +87,48 @@ func Test_getProxyMode(t *testing.T) { kernelCompat: true, expected: proxyModeIPTables, }, - { // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible + { // flag says ipvs, ipset version ok, kernel modules installed + flag: "ipvs", + kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, + expected: proxyModeIPVS, + }, + { // flag says ipvs, ipset version too low, fallback on iptables mode flag: "ipvs", + kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "0.0", iptablesVersion: iptables.MinCheckVersion, kernelCompat: true, expected: proxyModeIPTables, }, - { // specify ipvs, feature gateway disabled, iptables version too low + { // flag says ipvs, bad ipset version, fallback on iptables mode flag: "ipvs", + kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "a.b.c", + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, required kernel modules are not installed, fallback on iptables mode + flag: "ipvs", + kmods: []string{"foo", "bar", "baz"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, + iptablesVersion: iptables.MinCheckVersion, + kernelCompat: true, + expected: proxyModeIPTables, + }, + { // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode + flag: "ipvs", + kmods: []string{"foo", "bar", "baz"}, + ipsetVersion: ipvs.MinIPSetCheckVersion, iptablesVersion: "0.0.0", + kernelCompat: true, expected: proxyModeUserspace, }, - { // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible + { // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode flag: "ipvs", + kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, + ipsetVersion: "0.0", iptablesVersion: iptables.MinCheckVersion, kernelCompat: false, expected: proxyModeUserspace, @@ -117,131 +145,3 @@ func Test_getProxyMode(t *testing.T) { } } } - -// This is a coarse test, but it offers some modicum of confidence as the code is evolved. -func Test_getProxyModeEnableFeatureGateway(t *testing.T) { - // enable IPVS feature gateway - utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true") - - var cases = []struct { - flag string - iptablesVersion string - ipsetVersion string - kernelCompat bool - iptablesError error - ipsetError error - mods []string - expected string - }{ - { // flag says userspace - flag: "userspace", - expected: proxyModeUserspace, - }, - { // flag says iptables, error detecting version - flag: "iptables", - iptablesError: fmt.Errorf("oops!"), - expected: proxyModeUserspace, - }, - { // flag says iptables, version too low - flag: "iptables", - iptablesVersion: "0.0.0", - expected: proxyModeUserspace, - }, - { // flag says iptables, version ok, kernel not compatible - flag: "iptables", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: false, - expected: proxyModeUserspace, - }, - { // flag says iptables, version ok, kernel is compatible - flag: "iptables", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // detect, error - flag: "", - iptablesError: fmt.Errorf("oops!"), - expected: proxyModeUserspace, - }, - { // detect, version too low - flag: "", - iptablesVersion: "0.0.0", - expected: proxyModeUserspace, - }, - { // detect, version ok, kernel not compatible - flag: "", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: false, - expected: proxyModeUserspace, - }, - { // detect, version ok, kernel is compatible - flag: "", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // detect, version ok, kernel is compatible - flag: "", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, ipset version ok, kernel modules installed - flag: "ipvs", - mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, - ipsetVersion: ipvs.MinIPSetCheckVersion, - expected: proxyModeIPVS, - }, - { // flag says ipvs, ipset version too low, fallback on iptables mode - flag: "ipvs", - mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, - ipsetVersion: "0.0", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, bad ipset version, fallback on iptables mode - flag: "ipvs", - mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, - ipsetVersion: "a.b.c", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, required kernel modules are not installed, fallback on iptables mode - flag: "ipvs", - mods: []string{"foo", "bar", "baz"}, - ipsetVersion: ipvs.MinIPSetCheckVersion, - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode - flag: "ipvs", - mods: []string{"foo", "bar", "baz"}, - ipsetVersion: ipvs.MinIPSetCheckVersion, - iptablesVersion: "0.0.0", - kernelCompat: true, - expected: proxyModeUserspace, - }, - { // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode - flag: "ipvs", - mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, - ipsetVersion: "0.0", - iptablesVersion: iptables.MinCheckVersion, - kernelCompat: false, - expected: proxyModeUserspace, - }, - } - for i, c := range cases { - versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError} - kcompater := &fakeKernelCompatTester{c.kernelCompat} - ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError} - khandle := &fakeKernelHandler{c.mods} - r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater) - if r != c.expected { - t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) - } - } -} diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 86dd84ba94d..1ee6fddedd4 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -29,7 +29,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" "k8s.io/kubernetes/pkg/util/configz" utilpointer "k8s.io/kubernetes/pkg/util/pointer" @@ -189,8 +188,6 @@ conntrack: min: 1 tcpCloseWaitTimeout: 10s tcpEstablishedTimeout: 20s -featureGates: - SupportIPVSProxyMode: true healthzBindAddress: "%s" hostnameOverride: "foo" iptables: @@ -307,7 +304,7 @@ nodePortAddresses: TCPCloseWaitTimeout: &metav1.Duration{Duration: 10 * time.Second}, TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second}, }, - FeatureGates: map[string]bool{string(features.SupportIPVSProxyMode): true}, + FeatureGates: map[string]bool{}, HealthzBindAddress: tc.healthzBindAddress, HostnameOverride: "foo", IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 9abcbc22d91..44f00d3e885 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -126,13 +126,11 @@ if [ "${CLOUD_PROVIDER}" == "openstack" ]; then fi fi -# set feature gates if using ipvs mode +# load required kernel modules if proxy mode is set to "ipvs". if [ "${KUBE_PROXY_MODE}" == "ipvs" ]; then # If required kernel modules are not available, fall back to iptables. sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4 - if [[ $? -eq 0 ]]; then - FEATURE_GATES="${FEATURE_GATES},SupportIPVSProxyMode=true" - else + if [[ $? -ne 0 ]]; then echo "Required kernel modules for ipvs not found. Falling back to iptables mode." KUBE_PROXY_MODE=iptables fi diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index d773374ec43..b944a006717 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -211,7 +211,7 @@ const ( ResourceLimitsPriorityFunction utilfeature.Feature = "ResourceLimitsPriorityFunction" // owner: @m1093782566 - // beta: v1.9 + // GA: v1.11 // // Implement IPVS-based in-cluster service load balancing SupportIPVSProxyMode utilfeature.Feature = "SupportIPVSProxyMode" @@ -317,7 +317,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS BlockVolume: {Default: false, PreRelease: utilfeature.Alpha}, StorageObjectInUseProtection: {Default: true, PreRelease: utilfeature.GA}, ResourceLimitsPriorityFunction: {Default: false, PreRelease: utilfeature.Alpha}, - SupportIPVSProxyMode: {Default: true, PreRelease: utilfeature.Beta}, + SupportIPVSProxyMode: {Default: true, PreRelease: utilfeature.GA}, SupportPodPidsLimit: {Default: false, PreRelease: utilfeature.Alpha}, HyperVContainer: {Default: false, PreRelease: utilfeature.Alpha}, ScheduleDaemonSetPods: {Default: false, PreRelease: utilfeature.Alpha},