Merge pull request #52183 from MrHohn/kube-proxy-incluster-host

Automatic merge from submit-queue (batch tested with PRs 52883, 52183, 53915, 53848). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[GCE kube-up] Don't provision kubeconfig file for kube-proxy service account

**What this PR does / why we need it**:

Offloading the burden of provisioning kubeconfig file for kube-proxy service account from GCE startup scripts. This also helps us decoupling kube-proxy daemonset upgrade from node upgrade.

Previous attempt on https://github.com/kubernetes/kubernetes/pull/51172, using InClusterConfig for kube-proxy based on discussions on https://github.com/kubernetes/client-go/issues/281.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #NONE 

**Special notes for your reviewer**:
/assign @bowei @thockin 
cc @luxas @murali-reddy

**Release note**:

```release-note
NONE
```

cluster/addons/kube-proxy/kube-proxy-ds.yaml

@@ -37,8 +37,10 @@ spec:
         command:
         - /bin/sh
         - -c
-        - kube-proxy {{kubeconfig}} {{cluster_cidr}} --resource-container="" --oom-score-adj=-998 {{params}} 1>>/var/log/kube-proxy.log 2>&1
+        - kube-proxy {{cluster_cidr}} --resource-container="" --oom-score-adj=-998 {{params}} 1>>/var/log/kube-proxy.log 2>&1
-        {{container_env}}
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: {{kubernetes_service_host_env_value}}
         {{kube_cache_mutation_detector_env_name}}
           {{kube_cache_mutation_detector_env_value}}
         securityContext:
@@ -47,9 +49,6 @@ spec:
         - mountPath: /var/log
           name: varlog
           readOnly: false
-        - mountPath: /var/lib/kube-proxy/kubeconfig
-          name: kubeconfig
-          readOnly: false
         - mountPath: /run/xtables.lock
           name: xtables-lock
           readOnly: false
@@ -57,9 +56,6 @@ spec:
       - name: varlog
         hostPath:
           path: /var/log
-      - name: kubeconfig
-        hostPath:
-          path: /var/lib/kube-proxy/kubeconfig
       - name: xtables-lock
         hostPath:
           path: /run/xtables.lock

cluster/gce/configure-vm.sh

@@ -667,13 +667,12 @@ EOF
 #
 #  - When run as static pods, use the CA_CERT and KUBE_PROXY_TOKEN to generate a
 #    kubeconfig file for the kube-proxy to securely connect to the apiserver.
-#  - When run as a daemonset, generate a kubeconfig file specific to service account.
 function create-salt-kubeproxy-auth() {
   local -r kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
-  local kubeconfig_content=""
   if [ ! -e "${kube_proxy_kubeconfig_file}" ]; then
-    if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    mkdir -p /srv/salt-overlay/salt/kube-proxy
-      kubeconfig_content="\
+    (umask 077;
+        cat > "${kube_proxy_kubeconfig_file}" <<EOF
 apiVersion: v1
 kind: Config
 users:
@@ -689,33 +688,7 @@ contexts:
     cluster: local
     user: kube-proxy
   name: service-account-context
-current-context: service-account-context"
+current-context: service-account-context
-    else
-      # Generate kubeconfig specific to service account.
-      kubeconfig_content="\
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    server: https://${KUBERNETES_MASTER_NAME}
-  name: default
-contexts:
-- context:
-    cluster: default
-    namespace: default
-    user: default
-  name: default
-current-context: default
-users:
-- name: default
-  user:
-    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token"
-    fi
-    mkdir -p /srv/salt-overlay/salt/kube-proxy
-    (umask 077;
-        cat > "${kube_proxy_kubeconfig_file}" <<EOF
-${kubeconfig_content}
 EOF
 )
   fi
@@ -891,7 +864,9 @@ if [[ -z "${is_push}" ]]; then
   create-node-pki
   create-salt-pillar
   create-salt-kubelet-auth
-  create-salt-kubeproxy-auth
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    create-salt-kubeproxy-auth
+  fi
   download-release
   configure-salt
   remove-docker-artifacts

cluster/gce/container-linux/configure-helper.sh

@@ -385,30 +385,6 @@ current-context: service-account-context
 EOF
 }
 
-function create-kubeproxy-serviceaccount-kubeconfig {
-  echo "Creating kube-proxy serviceaccount kubeconfig file"
-  cat <<EOF >/var/lib/kube-proxy/kubeconfig
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    server: https://${KUBERNETES_MASTER_NAME}
-  name: default
-contexts:
-- context:
-    cluster: default
-    namespace: default
-    user: default
-  name: default
-current-context: default
-users:
-- name: default
-  user:
-    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-EOF
-}
-
 function create-kubecontrollermanager-kubeconfig {
   echo "Creating kube-controller-manager kubeconfig file"
   mkdir -p /etc/srv/kubernetes/kube-controller-manager
@@ -719,6 +695,7 @@ function prepare-kube-proxy-manifest-variables {
   sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
   sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
   sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
+  sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" ${src_file}
   if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
     sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
   fi
@@ -1499,8 +1476,6 @@ else
   create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}"
   if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
     create-kubeproxy-user-kubeconfig
-  else
-    create-kubeproxy-serviceaccount-kubeconfig
   fi
 fi
 

cluster/gce/gci/configure-helper.sh

@@ -727,30 +727,6 @@ current-context: service-account-context
 EOF
 }
 
-function create-kubeproxy-serviceaccount-kubeconfig {
-  echo "Creating kube-proxy serviceaccount kubeconfig file"
-  cat <<EOF >/var/lib/kube-proxy/kubeconfig
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    server: https://${KUBERNETES_MASTER_NAME}
-  name: default
-contexts:
-- context:
-    cluster: default
-    namespace: default
-    user: default
-  name: default
-current-context: default
-users:
-- name: default
-  user:
-    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-EOF
-}
-
 function create-kubecontrollermanager-kubeconfig {
   echo "Creating kube-controller-manager kubeconfig file"
   mkdir -p /etc/srv/kubernetes/kube-controller-manager
@@ -1119,6 +1095,7 @@ function prepare-kube-proxy-manifest-variables {
   sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
   sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
   sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
+  sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" ${src_file}
   if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
     sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
   fi
@@ -2009,8 +1986,6 @@ else
   create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
   if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
     create-kubeproxy-user-kubeconfig
-  else
-    create-kubeproxy-serviceaccount-kubeconfig
   fi
   if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
     create-node-problem-detector-kubeconfig

cmd/kube-proxy/app/BUILD

@@ -66,6 +66,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",

cmd/kube-proxy/app/server.go

@@ -41,6 +41,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/tools/record"
@@ -384,15 +385,19 @@ type ProxyServer struct {
 // createClients creates a kube client and an event client from the given config and masterOverride.
 // TODO remove masterOverride when CLI flags are removed.
 func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, v1core.EventsGetter, error) {
-	if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 {
+	var kubeConfig *rest.Config
-		glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
+	var err error
-	}
 
-	// This creates a client, first loading any specified kubeconfig
+	if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 {
-	// file, and then overriding the Master flag, if non-empty.
+		glog.Info("Neither kubeconfig file nor master URL was specified. Falling back to in-cluster config.")
-	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		kubeConfig, err = rest.InClusterConfig()
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.KubeConfigFile},
+	} else {
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig()
+		// This creates a client, first loading any specified kubeconfig
+		// file, and then overriding the Master flag, if non-empty.
+		kubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.KubeConfigFile},
+			&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig()
+	}
 	if err != nil {
 		return nil, nil, err
 	}