From 2eb24f9ae159bba9cfe535363ae77830457b32e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20K=C5=82obuszewski?= Date: Mon, 5 Feb 2018 13:13:22 +0100 Subject: [PATCH 1/3] Enable scaling fluentd-gcp resources using ScalingPolicy. See https://github.com/justinsb/scaler for more details about ScalingPolicy resource. --- .../addons/fluentd-gcp/fluentd-gcp-ds.yaml | 6 -- .../addons/fluentd-gcp/scaler-deployment.yaml | 34 +++++++++++ cluster/addons/fluentd-gcp/scaler-policy.yaml | 13 ++++ cluster/addons/fluentd-gcp/scaler-rbac.yaml | 48 +++++++++++++++ cluster/gce/config-default.sh | 6 +- cluster/gce/config-test.sh | 6 +- cluster/gce/gci/configure-helper.sh | 59 +++++++++++-------- 7 files changed, 134 insertions(+), 38 deletions(-) create mode 100644 cluster/addons/fluentd-gcp/scaler-deployment.yaml create mode 100644 cluster/addons/fluentd-gcp/scaler-policy.yaml create mode 100644 cluster/addons/fluentd-gcp/scaler-rbac.yaml diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 118de635a42..ee77e59154b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -31,12 +31,6 @@ spec: env: - name: FLUENTD_ARGS value: --no-supervisor -q - resources: - limits: - memory: 300Mi - requests: - cpu: 100m - memory: 200Mi volumeMounts: - name: varlog mountPath: /var/log diff --git a/cluster/addons/fluentd-gcp/scaler-deployment.yaml b/cluster/addons/fluentd-gcp/scaler-deployment.yaml new file mode 100644 index 00000000000..99ed101eecd --- /dev/null +++ b/cluster/addons/fluentd-gcp/scaler-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentd-gcp-scaler + namespace: kube-system + labels: + k8s-app: fluentd-gcp-scaler + version: v0.1 + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: fluentd-gcp-scaler + template: + metadata: + labels: + k8s-app: fluentd-gcp-scaler + spec: + serviceAccountName: fluentd-gcp-scaler + containers: + - name: fluentd-gcp-scaler + image: gcr.io/google-containers/fluentd-gcp-scaler:0.1 + command: + - /scaler.sh + - --ds-name=fluentd-gcp-v2.0.13 + - --scaling-policy=fluentd-gcp-scaling-policy + env: + # Defaults, used if no overrides are found in fluentd-gcp-scaling-policy + - name: CPU_REQUEST + value: 100m + - name: MEMORY_REQUEST + value: 200Mi + - name: MEMORY_LIMIT + value: 300Mi diff --git a/cluster/addons/fluentd-gcp/scaler-policy.yaml b/cluster/addons/fluentd-gcp/scaler-policy.yaml new file mode 100644 index 00000000000..792d100a0ee --- /dev/null +++ b/cluster/addons/fluentd-gcp/scaler-policy.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: scalingpolicies.scalingpolicy.kope.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: scalingpolicy.kope.io + version: v1alpha1 + names: + kind: ScalingPolicy + plural: scalingpolicies + scope: Namespaced diff --git a/cluster/addons/fluentd-gcp/scaler-rbac.yaml b/cluster/addons/fluentd-gcp/scaler-rbac.yaml new file mode 100644 index 00000000000..f9505f76dee --- /dev/null +++ b/cluster/addons/fluentd-gcp/scaler-rbac.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd-gcp-scaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system:fluentd-gcp-scaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "extensions" + resources: + - daemonsets + verbs: + - get + - patch +- apiGroups: + - "scalingpolicy.kope.io" + resources: + - scalingpolicies + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fluentd-gcp-scaler-binding + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system:fluentd-gcp-scaler +subjects: +- kind: ServiceAccount + name: fluentd-gcp-scaler + namespace: kube-system diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c3e59d0babb..863240d6fcb 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -353,9 +353,9 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then fi # Fluentd requirements -FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-300Mi}" -FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-100m}" -FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-200Mi}" +FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-}" +FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-}" +FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-}" # Heapster requirements HEAPSTER_GCP_BASE_MEMORY="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 93d254a1b87..1ccf0408cdd 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -388,9 +388,9 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then fi # Fluentd requirements -FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-300Mi}" -FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-100m}" -FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-200Mi}" +FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-}" +FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-}" +FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-}" # Heapster requirements HEAPSTER_GCP_BASE_MEMORY="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b80d868f261..7f368021bdc 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1979,50 +1979,57 @@ function copy-manifests { chmod 644 "${dst_dir}"/* } -# Fluentd manifest is modified using kubectl, which may not be available at -# this point. Run this as a background process. +# Fluentd resources are modified using ScalingPolicy CR, which may not be +# available at this point. Run this as a background process. function wait-for-apiserver-and-update-fluentd { - local -r fluentd_gcp_yaml="${1}" - - local modifying_flags="" + local any_overrides=false if [[ -n "${FLUENTD_GCP_MEMORY_LIMIT:-}" ]]; then - modifying_flags="${modifying_flags} --limits=memory=${FLUENTD_GCP_MEMORY_LIMIT}" + any_overrides=true fi - local request_resources="" if [[ -n "${FLUENTD_GCP_CPU_REQUEST:-}" ]]; then - request_resources="cpu=${FLUENTD_GCP_CPU_REQUEST}" + any_overrides=true fi if [[ -n "${FLUENTD_GCP_MEMORY_REQUEST:-}" ]]; then - if [[ -n "${request_resources}" ]]; then - request_resources="${request_resources}," - fi - request_resources="memory=${FLUENTD_GCP_MEMORY_REQUEST}" + any_overrides=true fi - if [[ -n "${request_resources}" ]]; then - modifying_flags="${modifying_flags} --requests=${request_resources}" + if ! $any_overrides; then + # Nothing to do here. + exit fi - until kubectl get nodes + # Wait until ScalingPolicy CRD is in place. + until kubectl get scalingpolicies do sleep 10 done - local -r temp_fluentd_gcp_yaml="${fluentd_gcp_yaml}.tmp" - if kubectl set resources --dry-run --local -f ${fluentd_gcp_yaml} ${modifying_flags} \ - --containers=fluentd-gcp -o yaml > ${temp_fluentd_gcp_yaml}; then - mv ${temp_fluentd_gcp_yaml} ${fluentd_gcp_yaml} - else - (echo "Failed to update fluentd resources. Used manifest:" && cat ${temp_fluentd_gcp_yaml}) >&2 - rm ${temp_fluentd_gcp_yaml} - fi + # Single-shot, not managed by addon manager. Can be later modified or removed + # at will. + cat < Date: Tue, 13 Feb 2018 10:23:58 +0100 Subject: [PATCH 2/3] Review #1 --- cluster/addons/fluentd-gcp/scaler-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/addons/fluentd-gcp/scaler-deployment.yaml b/cluster/addons/fluentd-gcp/scaler-deployment.yaml index 99ed101eecd..8129f7ef823 100644 --- a/cluster/addons/fluentd-gcp/scaler-deployment.yaml +++ b/cluster/addons/fluentd-gcp/scaler-deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: k8s-app: fluentd-gcp-scaler - version: v0.1 + version: v0.1.0 addonmanager.kubernetes.io/mode: Reconcile spec: selector: From 6a48e343f0e9efbb0e7f8ca7e55aa4f33cd9ec1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20K=C5=82obuszewski?= Date: Tue, 13 Feb 2018 10:57:29 +0100 Subject: [PATCH 3/3] Review #2 --- cluster/gce/gci/configure-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 7f368021bdc..05a2a3addca 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1998,7 +1998,7 @@ function wait-for-apiserver-and-update-fluentd { fi # Wait until ScalingPolicy CRD is in place. - until kubectl get scalingpolicies + until kubectl get scalingpolicies.scalingpolicy.kope.io do sleep 10 done