From 7820b054676a7b946a729102bc9540a735a72c16 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jun 2020 14:16:33 -0700 Subject: [PATCH 1/3] Separate network proxy flag for apiserver egress and starting pods --- cluster/gce/config-default.sh | 12 ++++++++++-- cluster/gce/gci/configure-helper.sh | 8 ++++---- cluster/gce/gci/configure-kubeapiserver.sh | 2 +- cluster/gce/util.sh | 13 +++++++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index b339684b73d..045c590382f 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -518,8 +518,16 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche export GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" export GCE_PRIVATE_CLUSTER_PORTS_PER_VM="${KUBE_GCE_PRIVATE_CLUSTER_PORTS_PER_VM:-}" -# Optional: Create apiserver konnectivity server and agent. -export ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" +# When KUBE_ENABLE_KONNECTIVITY_SERVICE is enabled, the two variables below will +# will default to true to enable the konnectivity network proxy and start the required pods. +# Their values can be overridden for more granular control of the proxy. + +# Optional: Whether to use konnectivity network proxy for all egress from apiserver. +export EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" +# Optional: Whether to start the konnectivity server and agent pods. +export RUN_KONNECTIVITY_SERVICE_PODS="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" +# Proxy Protocol Mode determines the protocol to use to communicate between apiserver and network proxy. +# Valid options are grpc and http-connect. Default is grpc. export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" # Optional: Enable Windows CSI-Proxy diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 02bc98c357c..a389e4e3932 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -922,7 +922,7 @@ contexts: name: webhook EOF fi - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then cat </etc/srv/kubernetes/egress_selector_configuration.yaml apiVersion: apiserver.k8s.io/v1beta1 @@ -2599,7 +2599,7 @@ EOF setup-node-termination-handler-manifest '' fi # Setting up the konnectivity-agent daemonset - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${RUN_KONNECTIVITY_SERVICE_PODS:-false}" == "true" ]]; then setup-addon-manifests "addons" "konnectivity-agent" setup-konnectivity-agent-manifest fi @@ -3028,7 +3028,7 @@ function main() { if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)" fi - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)" fi if [[ "${ENABLE_MONITORING_TOKEN:-false}" == "true" ]]; then @@ -3089,7 +3089,7 @@ function main() { fi source ${KUBE_BIN}/configure-kubeapiserver.sh start-kube-apiserver - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${RUN_KONNECTIVITY_SERVICE_PODS:-false}" == "true" ]]; then start-konnectivity-server fi start-kube-controller-manager diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index fa88954beaf..9aae5e71e93 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -341,7 +341,7 @@ function start-kube-apiserver { local csc_config_volume="" local default_konnectivity_socket_vol="" local default_konnectivity_socket_mnt="" - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 669d50ac480..bd4b314bb0e 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1506,9 +1506,14 @@ EOF MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}") EOF fi - if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then - cat >>"$file" <>$file <>$file < Date: Thu, 9 Jul 2020 08:52:28 -0700 Subject: [PATCH 2/3] Add SETUP_KONNECTIVITY_SERVICE flag --- cluster/gce/config-default.sh | 7 +++++-- cluster/gce/gci/configure-helper.sh | 4 ++-- cluster/gce/gci/configure-kubeapiserver.sh | 6 ++++-- cluster/gce/util.sh | 13 +++++++++---- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 045c590382f..c290e09cae8 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -518,12 +518,15 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche export GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" export GCE_PRIVATE_CLUSTER_PORTS_PER_VM="${KUBE_GCE_PRIVATE_CLUSTER_PORTS_PER_VM:-}" -# When KUBE_ENABLE_KONNECTIVITY_SERVICE is enabled, the two variables below will +# When KUBE_ENABLE_KONNECTIVITY_SERVICE is enabled, the three variables below will # will default to true to enable the konnectivity network proxy and start the required pods. # Their values can be overridden for more granular control of the proxy. +# Optional: Whether to do the setup for the konnectivity service +# Includes setting up kubeconfig, tokens, egress files, and firewall rules +export SETUP_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Optional: Whether to use konnectivity network proxy for all egress from apiserver. -export EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" +export EGRESS_VIA_KONNECTIVITY="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Optional: Whether to start the konnectivity server and agent pods. export RUN_KONNECTIVITY_SERVICE_PODS="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Proxy Protocol Mode determines the protocol to use to communicate between apiserver and network proxy. diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index a389e4e3932..dd62b861198 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -922,7 +922,7 @@ contexts: name: webhook EOF fi - if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then cat </etc/srv/kubernetes/egress_selector_configuration.yaml apiVersion: apiserver.k8s.io/v1beta1 @@ -3028,7 +3028,7 @@ function main() { if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)" fi - if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)" fi if [[ "${ENABLE_MONITORING_TOKEN:-false}" == "true" ]]; then diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index 9aae5e71e93..75201c9b39c 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -341,17 +341,19 @@ function start-kube-apiserver { local csc_config_volume="" local default_konnectivity_socket_vol="" local default_konnectivity_socket_mnt="" - if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," - params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" # UDS socket for communication between apiserver and konnectivity-server local default_konnectivity_socket_path="/etc/srv/kubernetes/konnectivity-server" default_konnectivity_socket_vol="{ \"name\": \"konnectivity-socket\", \"hostPath\": {\"path\": \"${default_konnectivity_socket_path}\", \"type\": \"DirectoryOrCreate\"}}," default_konnectivity_socket_mnt="{ \"name\": \"konnectivity-socket\", \"mountPath\": \"${default_konnectivity_socket_path}\", \"readOnly\": false}," fi + if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then + params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" + fi local container_env="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index bd4b314bb0e..a3dc3802d74 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1506,14 +1506,19 @@ EOF MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}") EOF fi - if [[ "${EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then cat >>$file <>$file <>$file < Date: Thu, 16 Jul 2020 11:43:34 -0700 Subject: [PATCH 3/3] Rename flags --- cluster/gce/config-default.sh | 6 +++--- cluster/gce/gci/configure-helper.sh | 8 ++++---- cluster/gce/gci/configure-kubeapiserver.sh | 2 +- cluster/gce/util.sh | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c290e09cae8..7a81d0195e7 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -519,16 +519,16 @@ export GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" export GCE_PRIVATE_CLUSTER_PORTS_PER_VM="${KUBE_GCE_PRIVATE_CLUSTER_PORTS_PER_VM:-}" # When KUBE_ENABLE_KONNECTIVITY_SERVICE is enabled, the three variables below will -# will default to true to enable the konnectivity network proxy and start the required pods. +# default to true to enable the konnectivity network proxy and start the required pods. # Their values can be overridden for more granular control of the proxy. # Optional: Whether to do the setup for the konnectivity service # Includes setting up kubeconfig, tokens, egress files, and firewall rules -export SETUP_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" +export PREPARE_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Optional: Whether to use konnectivity network proxy for all egress from apiserver. export EGRESS_VIA_KONNECTIVITY="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Optional: Whether to start the konnectivity server and agent pods. -export RUN_KONNECTIVITY_SERVICE_PODS="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" +export RUN_KONNECTIVITY_PODS="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-false}" # Proxy Protocol Mode determines the protocol to use to communicate between apiserver and network proxy. # Valid options are grpc and http-connect. Default is grpc. export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index dd62b861198..f707f1d7b9d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -922,7 +922,7 @@ contexts: name: webhook EOF fi - if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then cat </etc/srv/kubernetes/egress_selector_configuration.yaml apiVersion: apiserver.k8s.io/v1beta1 @@ -2599,7 +2599,7 @@ EOF setup-node-termination-handler-manifest '' fi # Setting up the konnectivity-agent daemonset - if [[ "${RUN_KONNECTIVITY_SERVICE_PODS:-false}" == "true" ]]; then + if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then setup-addon-manifests "addons" "konnectivity-agent" setup-konnectivity-agent-manifest fi @@ -3028,7 +3028,7 @@ function main() { if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)" fi - if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)" fi if [[ "${ENABLE_MONITORING_TOKEN:-false}" == "true" ]]; then @@ -3089,7 +3089,7 @@ function main() { fi source ${KUBE_BIN}/configure-kubeapiserver.sh start-kube-apiserver - if [[ "${RUN_KONNECTIVITY_SERVICE_PODS:-false}" == "true" ]]; then + if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then start-konnectivity-server fi start-kube-controller-manager diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index 75201c9b39c..f51e1e8caa1 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -341,7 +341,7 @@ function start-kube-apiserver { local csc_config_volume="" local default_konnectivity_socket_vol="" local default_konnectivity_socket_mnt="" - if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index a3dc3802d74..f9ec0b40f2f 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1506,9 +1506,9 @@ EOF MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}") EOF fi - if [[ "${SETUP_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then cat >>$file <>$file <