From d903d3285668858553fba8f89a69b26f0945f4da Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Thu, 31 May 2018 19:19:34 -0700 Subject: [PATCH 1/4] Add netd as an addon for GKE. --- cluster/gce/config-default.sh | 3 +++ cluster/gce/config-test.sh | 3 +++ cluster/gce/gci/configure-helper.sh | 16 ++++++++++++++++ cluster/gce/util.sh | 1 + 4 files changed, 23 insertions(+) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 95cc6a576cc..b393e848c60 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -349,6 +349,9 @@ STORAGE_BACKEND=${STORAGE_BACKEND:-} # Networking plugin specific settings. NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet +# Optional: Enable netd. +ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" + # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index bbc1c6e8dc0..9d533533906 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -359,6 +359,9 @@ STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet +# Optional: Enable netd. +ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" + # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 515bb2d22ee..3ab47122802 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2235,6 +2235,19 @@ EOF fi } +# Sets up the manifests of netd for k8s addons. +function setup-netd-manifest { + local -r netd_file="${dst_dir}/netd/netd.yaml" + mkdir -p "${dst_dir}/netd" + touch "${netd_file}" + if [ -n "${CUSTOM_NETD_YAML:-}" ]; then + # Replace with custom GKE netd deployment. + cat > "${netd_file}" < Date: Sun, 3 Jun 2018 01:35:27 -0700 Subject: [PATCH 2/4] support netd on k8s --- cluster/gce/config-default.sh | 1 + cluster/gce/config-test.sh | 1 + cluster/gce/util.sh | 30 ++++++++++++++++++++++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index b393e848c60..52cbb648564 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -351,6 +351,7 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet # Optional: Enable netd. ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" +CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 9d533533906..94e88d5a373 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -361,6 +361,7 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet # Optional: Enable netd. ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" +CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index ec125dfa383..7fd078ca850 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -530,6 +530,9 @@ function build-node-labels { if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${master}" != "true" ]]; then node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}" fi + if [[ "${ENABLE_NETD:-}" == "true" && "${master}" != "true" ]]; then + node_labels="${node_labels:+${node_labels},}beta.kubernetes.io/kube-netd-ready=true" + fi echo $node_labels } @@ -652,11 +655,19 @@ function construct-kubelet-flags { fi else # Otherwise use the configured value. - flags+=" --network-plugin=${NETWORK_PROVIDER}" + if [[ "${ENABLE_NETD:-}" == "true" && "${master}" != "true" ]]; then + flags+=" --network-plugin=cni" + else + flags+=" --network-plugin=${NETWORK_PROVIDER}" + fi fi fi if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then - flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" + if [[ "${ENABLE_NETD:-}" == "true" && "${master}" != "true" ]]; then + flags+=" --non-masquerade-cidr=0.0.0.0/0" + else + flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" + fi fi flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then @@ -850,7 +861,6 @@ RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-}) KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-}) -NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-}) NETWORK_POLICY_PROVIDER: $(yaml-quote ${NETWORK_POLICY_PROVIDER:-}) PREPULL_E2E_IMAGES: $(yaml-quote ${PREPULL_E2E_IMAGES:-}) HAIRPIN_MODE: $(yaml-quote ${HAIRPIN_MODE:-}) @@ -858,7 +868,6 @@ E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-}) KUBE_DOCKER_REGISTRY: $(yaml-quote ${KUBE_DOCKER_REGISTRY:-}) KUBE_ADDON_REGISTRY: $(yaml-quote ${KUBE_ADDON_REGISTRY:-}) MULTIZONE: $(yaml-quote ${MULTIZONE:-}) -NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-}) ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-}) ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) @@ -899,7 +908,20 @@ VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR}) KUBELET_ARGS: $(yaml-quote ${KUBELET_ARGS}) REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true) ENABLE_NETD: $(yaml-quote ${ENABLE_NETD:-false}) +CUSTOM_NETD_YAML: | +$(echo "${CUSTOM_NETD_YAML:-}" | sed -e "s/'/''/g") EOF + if [[ ${ENABLE_NETD:-} == "true" && "${master}" == "false" ]]; then + cat >>$file <>$file < Date: Sun, 3 Jun 2018 19:41:47 -0700 Subject: [PATCH 3/4] Code clean up --- cluster/gce/config-default.sh | 14 +++++++++---- cluster/gce/gci/configure-helper.sh | 2 +- cluster/gce/util.sh | 31 ++++++----------------------- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 52cbb648564..e6269e1a840 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -180,6 +180,16 @@ if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true" fi +# Optional: Enable netd. +ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" +CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" + +# To avoid running netd on a node that is not configured appropriately, +# label each Node so that the DaemonSet can run the Pods only on ready Nodes. +if [[ ${ENABLE_NETD:-} == "true" ]]; then + NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}beta.kubernetes.io/kube-netd-ready=true" +fi + # Enable metadata concealment by firewalling pod traffic to the metadata server # and run a proxy daemonset on nodes. # @@ -349,10 +359,6 @@ STORAGE_BACKEND=${STORAGE_BACKEND:-} # Networking plugin specific settings. NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet -# Optional: Enable netd. -ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" -CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" - # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 3ab47122802..4c21323b010 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2241,7 +2241,7 @@ function setup-netd-manifest { mkdir -p "${dst_dir}/netd" touch "${netd_file}" if [ -n "${CUSTOM_NETD_YAML:-}" ]; then - # Replace with custom GKE netd deployment. + # Replace with custom GCP netd deployment. cat > "${netd_file}" <>$file <>$file < Date: Sun, 3 Jun 2018 20:47:26 -0700 Subject: [PATCH 4/4] add NON_MASTER_NODE_LABELS to config-test.sh --- cluster/gce/config-test.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 94e88d5a373..5e0604a5849 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -212,6 +212,16 @@ NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes. NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}" +# Optional: Enable netd. +ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" +CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" + +# To avoid running netd on a node that is not configured appropriately, +# label each Node so that the DaemonSet can run the Pods only on ready Nodes. +if [[ ${ENABLE_NETD:-} == "true" ]]; then + NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}beta.kubernetes.io/kube-netd-ready=true" +fi + # To avoid running Calico on a node that is not configured appropriately, # label each Node so that the DaemonSet can run the Pods only on ready Nodes. if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then @@ -359,10 +369,6 @@ STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, kubenet -# Optional: Enable netd. -ENABLE_NETD="${KUBE_ENABLE_NETD:-false}" -CUSTOM_NETD_YAML="${KUBE_CUSTOM_NETD_YAML:-}" - # Network Policy plugin specific settings. NETWORK_POLICY_PROVIDER="${NETWORK_POLICY_PROVIDER:-none}" # calico