github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #38622 from Crassirostris/fluentd-liveness-probe

Browse Source 
Automatic merge from submit-queue

Add liveness probe for fluentd-gcp

It's known that fluentd can hung up during execution until manual restart.

Liveness probe fixes this problem in the following way: if no buffer chunks were sent or created in the last 5 minutes, fluentd is hanging and should be restarted.

CC @piosz
This commit is contained in:
Kubernetes Submit Queue 2016-12-22 02:43:28 -08:00 committed by GitHub
commit a30b6e2d16
2 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
View File

@ -42,6 +42,25 @@ spec:
readOnly: true
- name: libsystemddir
mountPath: /host/lib
# Liveness probe is aimed to help in situarions where fluentd
# silently hangs for no apparent reasons until manual restart.
# The idea of this probe is that if fluentd is not queueing or
# flushing chunks for 5 minutes, something is not right. If
# you want to change the fluentd configuration, reducing amount of
# logs fluentd collects, consider changing the threshold or turning
# liveness probe off completely.
livenessProbe:
initialDelaySeconds: 600
periodSeconds: 60
exec:
command:
- '/bin/sh'
- '-c'
- >
LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-600};
LAST_MODIFIED_DATE=`stat /var/log/fluentd-buffers | grep Modify | sed -r "s/Modify: (.*)/\1/"`;
LAST_MODIFIED_TIMESTAMP=`date -d "$LAST_MODIFIED_DATE" +%s`;
if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $LIVENESS_THRESHOLD_SECONDS` ]; then exit 1; fi;
nodeSelector:
alpha.kubernetes.io/fluentd-ds-ready: "true"
terminationGracePeriodSeconds: 30

19
cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
View File

@ -36,6 +36,25 @@ spec:
readOnly: true
- name: libsystemddir
mountPath: /host/lib
# Liveness probe is aimed to help in situarions where fluentd
# silently hangs for no apparent reasons until manual restart.
# The idea of this probe is that if fluentd is not queueing or
# flushing chunks for 5 minutes, something is not right. If
# you want to change the fluentd configuration, reducing amount of
# logs fluentd collects, consider changing the threshold or turning
# liveness probe off completely.
livenessProbe:
initialDelaySeconds: 600
periodSeconds: 60
exec:
command:
- '/bin/sh'
- '-c'
- >
LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-600};
LAST_MODIFIED_DATE=`stat /var/log/fluentd-buffers | grep Modify | sed -r "s/Modify: (.*)/\1/"`;
LAST_MODIFIED_TIMESTAMP=`date -d "$LAST_MODIFIED_DATE" +%s`;
if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $LIVENESS_THRESHOLD_SECONDS` ]; then exit 1; fi;
terminationGracePeriodSeconds: 30
volumes:
- name: varlog