From 89752337888d4ad6902b78f5440fd6297c8a9a3a Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 11:04:50 -0500 Subject: [PATCH 1/6] Finish migration of fluentd to daemonset --- cluster/addons/fluentd-elasticsearch/README.md | 4 ---- cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml | 2 -- cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml | 1 - cluster/gce/config-default.sh | 8 +------- cluster/gce/config-test.sh | 8 +------- 5 files changed, 2 insertions(+), 21 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/README.md b/cluster/addons/fluentd-elasticsearch/README.md index 0763b127656..36ed4028a63 100644 --- a/cluster/addons/fluentd-elasticsearch/README.md +++ b/cluster/addons/fluentd-elasticsearch/README.md @@ -52,10 +52,6 @@ Fluentd is deployed as a [DaemonSet][daemonSet] which spawns a pod on each node that reads logs, generated by kubelet, container runtime and containers and sends them to Elasticsearch. -**Note:** in order for Fluentd to work, every Kubernetes node must be labeled -with `beta.kubernetes.io/fluentd-ds-ready=true`, as otherwise the Fluentd -DaemonSet will ignore them. - Learn more in the [official Kubernetes documentation][k8sElasticsearchDocs]. ## Building diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index c3df2ac253d..e0b97e30060 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -95,8 +95,6 @@ spec: readOnly: true - name: config-volume mountPath: /etc/fluent/config.d - nodeSelector: - beta.kubernetes.io/fluentd-ds-ready: "true" terminationGracePeriodSeconds: 30 volumes: - name: varlog diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index c05de8f887e..95d48e92bc0 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -103,7 +103,6 @@ spec: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD nodeSelector: - beta.kubernetes.io/fluentd-ds-ready: "true" beta.kubernetes.io/os: linux terminationGracePeriodSeconds: 60 tolerations: diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 34753dc7a5a..28437f4724a 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -38,13 +38,7 @@ MASTER_ROOT_DISK_SIZE=${MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)} NODE_DISK_TYPE=${NODE_DISK_TYPE:-pd-standard} NODE_DISK_SIZE=${NODE_DISK_SIZE:-100GB} NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} - -# Historically fluentd was a manifest pod and then was migrated to DaemonSet. -# To avoid situation during cluster upgrade when there are two instances -# of fluentd running on a node, kubelet need to mark node on which -# fluentd is not running as a manifest pod with appropriate label. -# TODO(piosz): remove this in 1.8 -NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +NODE_LABELS="${KUBE_NODE_LABELS:-}" WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 2a424bbbbe6..a568df26c21 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -38,13 +38,7 @@ MASTER_ROOT_DISK_SIZE=${MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)} NODE_DISK_TYPE=${NODE_DISK_TYPE:-pd-standard} NODE_DISK_SIZE=${NODE_DISK_SIZE:-100GB} NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} - -# Historically fluentd was a manifest pod and then was migrated to DaemonSet. -# To avoid situation during cluster upgrade when there are two instances -# of fluentd running on a node, kubelet need to mark node on which -# fluentd is not running as a manifest pod with appropriate label. -# TODO(piosz): remove this in 1.8 -NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +NODE_LABELS="${KUBE_NODE_LABELS:-}" WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices From d6664a23652dfb48dd31edd6821668c32c3c585e Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 11:06:24 -0500 Subject: [PATCH 2/6] Prepare switch from beta.kubernetes.io/metadata-proxy-ready to cloud.google.com/metadata-proxy-ready --- cluster/addons/metadata-proxy/gce/metadata-proxy.yaml | 1 + cluster/gce/config-default.sh | 2 ++ cluster/gce/config-test.sh | 2 ++ 3 files changed, 5 insertions(+) diff --git a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml index 4c4c29f2d5e..92f28103f8b 100644 --- a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml +++ b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml @@ -88,6 +88,7 @@ spec: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD nodeSelector: + # TODO(liggitt): switch to cloud.google.com/metadata-proxy-ready=true in v1.16 beta.kubernetes.io/metadata-proxy-ready: "true" beta.kubernetes.io/os: linux terminationGracePeriodSeconds: 30 diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 28437f4724a..c13c33050bd 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -211,6 +211,8 @@ ENABLE_METADATA_CONCEALMENT="${ENABLE_METADATA_CONCEALMENT:-false}" # true, fals METADATA_CONCEALMENT_NO_FIREWALL="${METADATA_CONCEALMENT_NO_FIREWALL:-false}" # true, false if [[ ${ENABLE_METADATA_CONCEALMENT:-} == "true" ]]; then # Put the necessary label on the node so the daemonset gets scheduled. + NODE_LABELS="${NODE_LABELS},cloud.google.com/metadata-proxy-ready=true" + # TODO(liggitt): remove this in v1.16 NODE_LABELS="${NODE_LABELS},beta.kubernetes.io/metadata-proxy-ready=true" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_METADATA_CONCEALMENT METADATA_CONCEALMENT_NO_FIREWALL" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index a568df26c21..43c9f0a2613 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -251,6 +251,8 @@ ENABLE_METADATA_CONCEALMENT="${ENABLE_METADATA_CONCEALMENT:-true}" # true, false METADATA_CONCEALMENT_NO_FIREWALL="${METADATA_CONCEALMENT_NO_FIREWALL:-false}" # true, false if [[ ${ENABLE_METADATA_CONCEALMENT:-} == "true" ]]; then # Put the necessary label on the node so the daemonset gets scheduled. + NODE_LABELS="${NODE_LABELS},cloud.google.com/metadata-proxy-ready=true" + # TODO(liggitt): remove this in v1.16 NODE_LABELS="${NODE_LABELS},beta.kubernetes.io/metadata-proxy-ready=true" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_METADATA_CONCEALMENT METADATA_CONCEALMENT_NO_FIREWALL" From 943b32a2895c26ac53ab30820d1569709608f1e8 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 11:36:08 -0500 Subject: [PATCH 3/6] Prepare switch from beta.kubernetes.io/kube-proxy-ds-ready to node.kubernetes.io/kube-proxy-ds-ready --- cluster/addons/kube-proxy/kube-proxy-ds.yaml | 1 + cluster/gce/util.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml index e082ca4d16b..fb28b02a4c8 100644 --- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -27,6 +27,7 @@ spec: priorityClassName: system-node-critical hostNetwork: true nodeSelector: + # TODO(liggitt): switch to node.kubernetes.io/kube-proxy-ds-ready in 1.16 beta.kubernetes.io/kube-proxy-ds-ready: "true" tolerations: - operator: "Exists" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 99ea09119fc..3f5aeec7ba3 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -590,7 +590,8 @@ function build-linux-node-labels { if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then # Add kube-proxy daemonset label to node to avoid situation during cluster # upgrade/downgrade when there are two instances of kube-proxy running on a node. - node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + # TODO(liggitt): drop beta.kubernetes.io/kube-proxy-ds-ready in 1.16 + node_labels="node.kubernetes.io/kube-proxy-ds-ready=true,beta.kubernetes.io/kube-proxy-ds-ready=true" fi if [[ -n "${NODE_LABELS:-}" ]]; then node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" From ca592a91d643201674dd2f4a8dca802e639adaac Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 11:50:19 -0500 Subject: [PATCH 4/6] Update timeline for kubelet label restrictions --- cmd/kubelet/app/options/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 8c386bbdc06..b0995804bc0 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -263,9 +263,9 @@ func ValidateKubeletFlags(f *KubeletFlags) error { } } if len(unknownLabels) > 0 { - // TODO(liggitt): in 1.15, return an error + // TODO(liggitt): in 1.16, return an error klog.Warningf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v", unknownLabels.List()) - klog.Warningf("in 1.15, --node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) + klog.Warningf("in 1.16, --node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) } return nil From 0174e043c5c0baac7d9d6da0e20cb2bbf01583dd Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 12:11:44 -0500 Subject: [PATCH 5/6] Prepare switch from beta.kubernetes.io/masq-agent-ds-ready to node.kubernetes.io/masq-agent-ds-ready --- cluster/addons/ip-masq-agent/ip-masq-agent.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml index b0ee2e7dea1..39f709f7981 100644 --- a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml +++ b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml @@ -45,6 +45,7 @@ spec: - name: config mountPath: /etc/config nodeSelector: + # TODO(liggitt): switch to node.kubernetes.io/masq-agent-ds-ready in 1.16 beta.kubernetes.io/masq-agent-ds-ready: "true" volumes: - name: config From e1db43ad1b52f17067b3bf23fb139aea98447938 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 22 Feb 2019 14:22:21 -0500 Subject: [PATCH 6/6] Label old nodes with legacy addon labels --- cluster/gce/gci/configure-helper.sh | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index a2e429f6a44..448b96f6da4 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1927,6 +1927,35 @@ function setup-etcd-encryption { fi } +# Updates node labels used by addons. +function update-legacy-addon-node-labels() { + # need kube-apiserver to be ready + until kubectl get nodes; do + sleep 5 + done + update-node-label "beta.kubernetes.io/metadata-proxy-ready=true,cloud.google.com/metadata-proxy-ready!=true" "cloud.google.com/metadata-proxy-ready=true" + update-node-label "beta.kubernetes.io/kube-proxy-ds-ready=true,node.kubernetes.io/kube-proxy-ds-ready!=true" "node.kubernetes.io/kube-proxy-ds-ready=true" + update-node-label "beta.kubernetes.io/masq-agent-ds-ready=true,node.kubernetes.io/masq-agent-ds-ready!=true" "node.kubernetes.io/masq-agent-ds-ready=true" +} + +# A helper function for labeling all nodes matching a given selector. +# Runs: kubectl label --overwrite nodes -l "${1}" "${2}" +# Retries on failure +# +# $1: label selector of nodes +# $2: label to apply +function update-node-label() { + local selector="$1" + local label="$2" + local retries=5 + until (( retries == 0 )); do + if kubectl label --overwrite nodes -l "${selector}" "${label}"; then + break + fi + (( retries-- )) + sleep 3 + done +} # Applies encryption provider config. # This function may be triggered in two scenarios: @@ -2899,6 +2928,7 @@ function main() { start-kube-addons start-cluster-autoscaler start-lb-controller + update-legacy-addon-node-labels & apply-encryption-config & else if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then