mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 03:11:40 +00:00
Use fluentbit as logging agent. Replace stackdriver
This commit is contained in:
Barni S
parent
e3d7d067ab
commit
bbe7b2ebed
@ -148,7 +148,7 @@ try {
|
||||
Configure-GcePdTools
|
||||
Configure-Kubelet
|
||||
|
||||
# Even if Stackdriver is already installed, the function will still [re]start the service.
|
||||
# Even if Logging agent is already installed, the function will still [re]start the service.
|
||||
if (IsLoggingEnabled $kube_env) {
|
||||
Install-LoggingAgent
|
||||
Configure-LoggingAgent
|
||||
|
||||
@ -1529,349 +1529,419 @@ function Start_Containerd {
|
||||
Log-Output "Starting containerd service"
|
||||
Start-Service containerd
|
||||
}
|
||||
|
||||
# TODO(pjh): move the Stackdriver logging agent code below into a separate
|
||||
# TODO(pjh): move the logging agent code below into a separate
|
||||
# module; it was put here temporarily to avoid disrupting the file layout in
|
||||
# the K8s release machinery.
|
||||
$STACKDRIVER_VERSION = 'v1-11'
|
||||
$STACKDRIVER_ROOT = 'C:\Program Files (x86)\Stackdriver'
|
||||
$LOGGINGAGENT_VERSION = '1.6.0'
|
||||
$LOGGINGAGENT_ROOT = 'C:\fluent-bit'
|
||||
$LOGGINGAGENT_SERVICE = 'fluent-bit'
|
||||
$LOGGINGAGENT_CMDLINE = '*fluent-bit.exe*'
|
||||
|
||||
# Restarts the Stackdriver logging agent, or starts it if it is not currently
|
||||
# running. A standard `Restart-Service StackdriverLogging` may fail because
|
||||
# StackdriverLogging sometimes is unstoppable, so this function works around it
|
||||
# by killing the processes.
|
||||
$LOGGINGEXPORTER_VERSION = 'v0.10.3'
|
||||
$LOGGINGEXPORTER_ROOT = 'C:\flb-exporter'
|
||||
$LOGGINGEXPORTER_SERVICE = 'flb-exporter'
|
||||
$LOGGINGEXPORTER_CMDLINE = '*flb-exporter.exe*'
|
||||
|
||||
# Restart Logging agent or starts it if it is not currently running
|
||||
function Restart-LoggingAgent {
|
||||
Stop-Service -NoWait -ErrorAction Ignore StackdriverLogging
|
||||
Restart-LogService $LOGGINGEXPORTER_SERVICE $LOGGINGEXPORTER_CMDLINE
|
||||
Restart-LogService $LOGGINGAGENT_SERVICE $LOGGINGAGENT_CMDLINE
|
||||
}
|
||||
|
||||
# Restarts the service, or starts it if it is not currently
|
||||
# running. A standard `Restart-Service` may fail because
|
||||
# the process is sometimes unstoppable, so this function works around it
|
||||
# by killing the processes.
|
||||
function Restart-LogService([string]$service, [string]$cmdline) {
|
||||
Stop-Service -NoWait -ErrorAction Ignore $service
|
||||
|
||||
# Wait (if necessary) for service to stop.
|
||||
$timeout = 10
|
||||
$stopped = (Get-service StackdriverLogging).Status -eq 'Stopped'
|
||||
$stopped = (Get-service $service).Status -eq 'Stopped'
|
||||
for ($i = 0; $i -lt $timeout -and !($stopped); $i++) {
|
||||
Start-Sleep 1
|
||||
$stopped = (Get-service StackdriverLogging).Status -eq 'Stopped'
|
||||
$stopped = (Get-service $service).Status -eq 'Stopped'
|
||||
}
|
||||
|
||||
if ((Get-service StackdriverLogging).Status -ne 'Stopped') {
|
||||
if ((Get-service $service).Status -ne 'Stopped') {
|
||||
# Force kill the processes.
|
||||
Stop-Process -Force -PassThru -Id (Get-WmiObject win32_process |
|
||||
Where CommandLine -Like '*Stackdriver/logging*').ProcessId
|
||||
Where CommandLine -Like $cmdline).ProcessId
|
||||
|
||||
# Wait until process has stopped.
|
||||
$waited = 0
|
||||
$log_period = 10
|
||||
$timeout = 60
|
||||
while ((Get-service StackdriverLogging).Status -ne 'Stopped' -and $waited -lt $timeout) {
|
||||
while ((Get-service $service).Status -ne 'Stopped' -and $waited -lt $timeout) {
|
||||
Start-Sleep 1
|
||||
$waited++
|
||||
|
||||
if ($waited % $log_period -eq 0) {
|
||||
Log-Output "Waiting for StackdriverLogging service to stop"
|
||||
Log-Output "Waiting for ${service} service to stop"
|
||||
}
|
||||
}
|
||||
|
||||
# Timeout occurred
|
||||
if ($waited -ge $timeout) {
|
||||
Throw ("Timeout while waiting for StackdriverLogging service to stop")
|
||||
Throw ("Timeout while waiting for ${service} service to stop")
|
||||
}
|
||||
}
|
||||
|
||||
Start-Service StackdriverLogging
|
||||
Start-Service $service
|
||||
}
|
||||
|
||||
# Check whether the logging agent is installed by whether it's registered as service
|
||||
function IsLoggingAgentInstalled {
|
||||
$stackdriver_status = (Get-Service StackdriverLogging -ErrorAction Ignore).Status
|
||||
return -not [string]::IsNullOrEmpty($stackdriver_status)
|
||||
$logging_status = (Get-Service $LOGGINGAGENT_SERVICE -ErrorAction Ignore).Status
|
||||
return -not [string]::IsNullOrEmpty($logging_status)
|
||||
}
|
||||
|
||||
# Clean up the logging agent's registry key and root folder if they exist from a prior installation.
|
||||
# Try to uninstall it first, if it failed, remove the registry key at least,
|
||||
# as the registry key will block the silent installation later on.
|
||||
function Cleanup-LoggingAgent {
|
||||
# For 64 bits app, the registry path is 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall'
|
||||
# for 32 bits app, it's 'HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall'
|
||||
# StackdriverLogging is installed as 32 bits app
|
||||
$x32_app_reg = 'HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall'
|
||||
$uninstall_string = (Get-ChildItem $x32_app_reg | Get-ItemProperty | Where-Object {$_.DisplayName -match "Stackdriver"}).UninstallString
|
||||
if (-not [string]::IsNullOrEmpty($uninstall_string)) {
|
||||
try {
|
||||
Start-Process -FilePath "$uninstall_string" -ArgumentList "/S" -Wait
|
||||
} catch {
|
||||
Log-Output "Exception happens during uninstall logging agent, so remove the registry key at least"
|
||||
Remove-Item -Path "$x32_app_reg\GoogleStackdriverLoggingAgent\"
|
||||
}
|
||||
}
|
||||
|
||||
# If we chose reboot after uninstallation, the root folder would be clean.
|
||||
# But since we couldn't reboot, so some files & folders would be left there,
|
||||
# which could block the re-installation later on, so clean it up
|
||||
if(Test-Path $STACKDRIVER_ROOT){
|
||||
Remove-Item -Force -Recurse $STACKDRIVER_ROOT
|
||||
}
|
||||
}
|
||||
|
||||
# Installs the Stackdriver logging agent according to
|
||||
# https://cloud.google.com/logging/docs/agent/installation.
|
||||
# TODO(yujuhong): Update to a newer Stackdriver agent once it is released to
|
||||
# support kubernetes metadata properly. The current version does not recognizes
|
||||
# the local resource key "logging.googleapis.com/local_resource_id", and fails
|
||||
# to label namespace, pod and container names on the logs.
|
||||
# Installs the logging agent according to https://docs.fluentbit.io/manual/installation/windows#
|
||||
# Also installs fluent bit stackdriver exporter
|
||||
function Install-LoggingAgent {
|
||||
# Remove the existing storage.json file if it exists. This is a workaround
|
||||
# for the bug where the logging agent cannot start up if the file is
|
||||
# corrupted.
|
||||
Remove-Item `
|
||||
-Force `
|
||||
-ErrorAction Ignore `
|
||||
("$STACKDRIVER_ROOT\LoggingAgent\Main\pos\winevtlog.pos\worker0\" +
|
||||
"storage.json")
|
||||
|
||||
if (IsLoggingAgentInstalled) {
|
||||
# Note: we should reinstall the Stackdriver agent if $REDO_STEPS is true
|
||||
# Note: we should reinstall the agent if $REDO_STEPS is true
|
||||
# here, but we don't know how to run the installer without it prompting
|
||||
# when Stackdriver is already installed. We dumped the strings in the
|
||||
# when logging agent is already installed. We dumped the strings in the
|
||||
# installer binary and searched for flags to do this but found nothing. Oh
|
||||
# well.
|
||||
Log-Output ("Skip: Stackdriver logging agent is already installed")
|
||||
Log-Output ("Skip: Fluentbit logging agent is already installed")
|
||||
return
|
||||
}
|
||||
|
||||
# After a crash, the StackdriverLogging service could be missing, but its files will still be present
|
||||
Cleanup-LoggingAgent
|
||||
|
||||
$url = ("https://storage.googleapis.com/gke-release/winnode/stackdriver/" +
|
||||
"StackdriverLogging-${STACKDRIVER_VERSION}.exe")
|
||||
$tmp_dir = 'C:\stackdriver_tmp'
|
||||
New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
|
||||
$installer_file = "${tmp_dir}\StackdriverLogging-${STACKDRIVER_VERSION}.exe"
|
||||
MustDownload-File -OutFile $installer_file -URLs $url
|
||||
|
||||
# Start the installer silently. This automatically starts the
|
||||
# "StackdriverLogging" service.
|
||||
Log-Output 'Invoking Stackdriver installer'
|
||||
Start-Process $installer_file -ArgumentList "/S" -Wait
|
||||
|
||||
# Install the record-reformer plugin.
|
||||
Start-Process "$STACKDRIVER_ROOT\LoggingAgent\Main\bin\fluent-gem" `
|
||||
-ArgumentList "install","fluent-plugin-record-reformer" `
|
||||
-Wait
|
||||
|
||||
# Install the multi-format-parser plugin.
|
||||
Start-Process "$STACKDRIVER_ROOT\LoggingAgent\Main\bin\fluent-gem" `
|
||||
-ArgumentList "install","fluent-plugin-multi-format-parser" `
|
||||
-Wait
|
||||
|
||||
Remove-Item -Force -Recurse $tmp_dir
|
||||
DownloadAndInstall-LoggingAgents
|
||||
Create-LoggingAgentServices
|
||||
}
|
||||
|
||||
# Writes the logging configuration file for Stackdriver. Restart-LoggingAgent
|
||||
function DownloadAndInstall-LoggingAgents {
|
||||
# Install Logging agent if not present
|
||||
if (ShouldWrite-File $LOGGINGAGENT_ROOT\td-agent-bit-${LOGGINGAGENT_VERSION}-win64) {
|
||||
$install_dir = 'C:\flb-installers'
|
||||
$url = ("https://storage.googleapis.com/gke-release/winnode/fluentbit/td-agent-bit-${LOGGINGAGENT_VERSION}-win64.zip")
|
||||
|
||||
Log-Output 'Downloading Logging agent'
|
||||
New-Item $install_dir -ItemType 'directory' -Force | Out-Null
|
||||
MustDownload-File -OutFile $install_dir\td.zip -URLs $url
|
||||
|
||||
cd $install_dir
|
||||
Log-Output 'Extracting Logging agent'
|
||||
Expand-Archive td.zip
|
||||
mv .\td\td-agent-bit-${LOGGINGAGENT_VERSION}-win64\ $LOGGINGAGENT_ROOT
|
||||
Remove-Item -Force -Recurse $install_dir
|
||||
}
|
||||
|
||||
# Download Logging exporter if needed
|
||||
if (ShouldWrite-File $LOGGINGEXPORTER_ROOT\flb-exporter.exe) {
|
||||
Log-Output 'Downloading logging exporter'
|
||||
New-Item $LOGGINGEXPORTER_ROOT -ItemType 'directory' -Force | Out-Null
|
||||
MustDownload-File `
|
||||
-OutFile $LOGGINGEXPORTER_ROOT\flb-exporter.exe `
|
||||
-URLs 'https://storage.googleapis.com/gke-release/winnode/fluentbit-exporter/${LOGGINGEXPORTER_VERSION}/flb-exporter-${LOGGINGEXPORTER_VERSION}.exe'
|
||||
}
|
||||
}
|
||||
|
||||
function Create-LoggingAgentServices {
|
||||
cd $LOGGINGAGENT_ROOT
|
||||
|
||||
Log-Output 'Creating service: ${LOGGINGAGENT_SERVICE}'
|
||||
sc.exe create $LOGGINGAGENT_SERVICE binpath= "${LOGGINGAGENT_ROOT}\bin\fluent-bit.exe -c \fluent-bit\conf\fluent-bit.conf"
|
||||
sc.exe failure $LOGGINGAGENT_SERVICE reset= 30 actions= restart/5000
|
||||
sc.exe query $LOGGINGAGENT_SERVICE
|
||||
|
||||
Log-Output 'Creating service: ${LOGGINGEXPORTER_SERVICE}'
|
||||
sc.exe create $LOGGINGEXPORTER_SERVICE binpath= "${LOGGINGEXPORTER_ROOT}\flb-exporter.exe --kubernetes-separator=_ --stackdriver-resource-model=k8s --enable-pod-label-discovery --logtostderr --winsvc --pod-label-dot-replacement=_"
|
||||
sc.exe failure $LOGGINGEXPORTER_SERVICE reset= 30 actions= restart/5000
|
||||
sc.exe query $LOGGINGEXPORTER_SERVICE
|
||||
}
|
||||
|
||||
# Writes the logging configuration file for Logging agent. Restart-LoggingAgent
|
||||
# should then be called to pick up the new configuration.
|
||||
function Configure-LoggingAgent {
|
||||
$fluentd_config_dir = "$STACKDRIVER_ROOT\LoggingAgent\config.d"
|
||||
$fluentd_config_file = "$fluentd_config_dir\k8s_containers.conf"
|
||||
$fluentbit_config_file = "$LOGGINGAGENT_ROOT\conf\fluent-bit.conf"
|
||||
$FLUENTBIT_CONFIG | Out-File -FilePath $fluentbit_config_file -Encoding ASCII
|
||||
Log-Output "Wrote logging config to $fluentbit_config_file"
|
||||
|
||||
# Create a configuration file for kubernetes containers.
|
||||
# The config.d directory should have already been created automatically, but
|
||||
# try creating again just in case.
|
||||
New-Item $fluentd_config_dir -ItemType 'directory' -Force | Out-Null
|
||||
|
||||
$config = $FLUENTD_CONFIG.replace('NODE_NAME', (hostname))
|
||||
$config | Out-File -FilePath $fluentd_config_file -Encoding ASCII
|
||||
Log-Output "Wrote fluentd logging config to $fluentd_config_file"
|
||||
$fluentbit_parser_file = "$LOGGINGAGENT_ROOT\conf\parsers.conf"
|
||||
$PARSERS_CONFIG | Out-File -FilePath $fluentbit_parser_file -Encoding ASCII
|
||||
Log-Output "Wrote logging config to $fluentbit_parser_file"
|
||||
}
|
||||
|
||||
# The NODE_NAME placeholder must be replaced with the node's name (hostname).
|
||||
$FLUENTD_CONFIG = @'
|
||||
# This configuration file for Fluentd is used to watch changes to kubernetes
|
||||
# container logs in the directory /var/lib/docker/containers/ and submit the
|
||||
# log records to Google Cloud Logging using the cloud-logging plugin.
|
||||
#
|
||||
# Example
|
||||
# Fluentbit main config file
|
||||
$FLUENTBIT_CONFIG = @'
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Grace 120
|
||||
Log_Level debug
|
||||
Log_File /var/log/fluentbit.log
|
||||
Daemon off
|
||||
Parsers_File parsers.conf
|
||||
HTTP_Server off
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_PORT 2020
|
||||
plugins_file plugins.conf
|
||||
|
||||
# Storage
|
||||
# =======
|
||||
# A line in the Docker log file might look like this JSON:
|
||||
# Fluent Bit can use memory and filesystem buffering based mechanisms
|
||||
#
|
||||
# {"log":"2014/09/25 21:15:03 Got request with path wombat\\n",
|
||||
# "stream":"stderr",
|
||||
# "time":"2014-09-25T21:15:03.499185026Z"}
|
||||
# - https://docs.fluentbit.io/manual/administration/buffering-and-storage
|
||||
#
|
||||
# The original tag is derived from the log file's location.
|
||||
# For example a Docker container's logs might be in the directory:
|
||||
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
|
||||
# and in the file:
|
||||
# 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
||||
# where 997599971ee6... is the Docker ID of the running container.
|
||||
# The Kubernetes kubelet makes a symbolic link to this file on the host
|
||||
# machine in the /var/log/containers directory which includes the pod name,
|
||||
# the namespace name and the Kubernetes container name:
|
||||
# synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# ->
|
||||
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
||||
# The /var/log directory on the host is mapped to the /var/log directory in the container
|
||||
# running this instance of Fluentd and we end up collecting the file:
|
||||
# /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# This results in the tag:
|
||||
# var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# where 'synthetic-logger-0.25lps-pod' is the pod name, 'default' is the
|
||||
# namespace name, 'synth-lgr' is the container name and '997599971ee6..' is
|
||||
# the container ID.
|
||||
# The record reformer is used to extract pod_name, namespace_name and
|
||||
# container_name from the tag and set them in a local_resource_id in the
|
||||
# format of:
|
||||
# 'k8s_container.<NAMESPACE_NAME>.<POD_NAME>.<CONTAINER_NAME>'.
|
||||
# The reformer also changes the tags to 'stderr' or 'stdout' based on the
|
||||
# value of 'stream'.
|
||||
# local_resource_id is later used by google_cloud plugin to determine the
|
||||
# monitored resource to ingest logs against.
|
||||
# storage metrics
|
||||
# ---------------
|
||||
# publish storage pipeline metrics in '/api/v1/storage'. The metrics are
|
||||
# exported only if the 'http_server' option is enabled.
|
||||
#
|
||||
# storage.metrics on
|
||||
|
||||
# storage.path
|
||||
# ------------
|
||||
# absolute file system path to store filesystem data buffers (chunks).
|
||||
#
|
||||
# storage.path /tmp/storage
|
||||
|
||||
# storage.sync
|
||||
# ------------
|
||||
# configure the synchronization mode used to store the data into the
|
||||
# filesystem. It can take the values normal or full.
|
||||
#
|
||||
# storage.sync normal
|
||||
|
||||
# storage.checksum
|
||||
# ----------------
|
||||
# enable the data integrity check when writing and reading data from the
|
||||
# filesystem. The storage layer uses the CRC32 algorithm.
|
||||
#
|
||||
# storage.checksum off
|
||||
|
||||
# storage.backlog.mem_limit
|
||||
# -------------------------
|
||||
# if storage.path is set, Fluent Bit will look for data chunks that were
|
||||
# not delivered and are still in the storage layer, these are called
|
||||
# backlog data. This option configure a hint of maximum value of memory
|
||||
# to use when processing these records.
|
||||
#
|
||||
# storage.backlog.mem_limit 5M
|
||||
|
||||
|
||||
[INPUT]
|
||||
Name winlog
|
||||
Interval_Sec 2
|
||||
# Channels Setup,Windows PowerShell
|
||||
Channels application,system,security
|
||||
Tag winevent.raw
|
||||
DB winlog.sqlite #
|
||||
|
||||
|
||||
# Json Log Example:
|
||||
# {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
|
||||
# CRI Log Example:
|
||||
# 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
|
||||
<source>
|
||||
@type tail
|
||||
path /var/log/containers/*.log
|
||||
pos_file /var/log/gcp-containers.log.pos
|
||||
# Tags at this point are in the format of:
|
||||
# reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log
|
||||
tag reform.*
|
||||
read_from_head true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format json
|
||||
time_key time
|
||||
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
||||
</pattern>
|
||||
<pattern>
|
||||
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
|
||||
time_format %Y-%m-%dT%H:%M:%S.%N%:z
|
||||
</pattern>
|
||||
</parse>
|
||||
</source>
|
||||
[INPUT]
|
||||
Name tail
|
||||
Alias kube_containers
|
||||
Tag kube_<namespace_name>_<pod_name>_<container_name>
|
||||
Tag_Regex (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-
|
||||
Path /var/log/containers/*.log
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 5
|
||||
DB flb_kube.db
|
||||
|
||||
# Settings from fluentd missing here.
|
||||
# tag reform.*
|
||||
# format json
|
||||
# time_key time
|
||||
# time_format %Y-%m-%dT%H:%M:%S.%NZ
|
||||
|
||||
|
||||
# Example:
|
||||
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
|
||||
<source>
|
||||
@type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /etc/kubernetes/logs/kubelet.log
|
||||
pos_file /etc/kubernetes/logs/gcp-kubelet.log.pos
|
||||
tag kubelet
|
||||
</source>
|
||||
[INPUT]
|
||||
Name tail
|
||||
Alias kubelet
|
||||
Tag kubelet
|
||||
#Multiline on
|
||||
#Multiline_Flush 5
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 5
|
||||
Path /etc/kubernetes/logs/kubelet.log
|
||||
DB /etc/kubernetes/logs/gcp-kubelet.db
|
||||
|
||||
# Copied from fluentbit config. How is this used ? In match stages ?
|
||||
Parser_Firstline /^\w\d{4}/
|
||||
Parser_1 ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
|
||||
# missing from fluentbit
|
||||
# time_format %m%d %H:%M:%S.%N
|
||||
|
||||
# Example:
|
||||
# I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
|
||||
<source>
|
||||
@type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /etc/kubernetes/logs/kube-proxy.log
|
||||
pos_file /etc/kubernetes/logs/gcp-kube-proxy.log.pos
|
||||
tag kube-proxy
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I0928 03:15:50.440223 4880 main.go:51] Starting CSI-Proxy Server ...
|
||||
<source>
|
||||
@type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /etc/kubernetes/logs/csi-proxy.log
|
||||
pos_file /etc/kubernetes/logs/gcp-csi-proxy.log.pos
|
||||
tag csi-proxy
|
||||
</source>
|
||||
[INPUT]
|
||||
Name tail
|
||||
Alias csi-proxy
|
||||
Tag csi-proxy
|
||||
#Multiline on
|
||||
#Multiline_Flush 5
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 5
|
||||
Path /etc/kubernetes/logs/csi-proxy.log
|
||||
DB /etc/kubernetes/logs/gcp-csi-proxy.db
|
||||
|
||||
# Copied from fluentbit config. How is this used ? In match stages ?
|
||||
Parser_Firstline /^\w\d{4}/
|
||||
Parser_1 ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
|
||||
# missing from fluentbit
|
||||
# time_format %m%d %H:%M:%S.%N
|
||||
|
||||
# Example:
|
||||
# time="2019-12-10T21:27:59.836946700Z" level=info msg="loading plugin \"io.containerd.grpc.v1.cri\"..." type=io.containerd.grpc.v1
|
||||
<source>
|
||||
@type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^time=/
|
||||
format1 /^time="(?<time>[^ ]*)" level=(?<severity>\w*) (?<message>.*)/
|
||||
time_format %Y-%m-%dT%H:%M:%S.%N%z
|
||||
path /etc/kubernetes/logs/containerd.log
|
||||
pos_file /etc/kubernetes/logs/gcp-containerd.log.pos
|
||||
tag container-runtime
|
||||
</source>
|
||||
[INPUT]
|
||||
Name tail
|
||||
Alias container-runtime
|
||||
Tag container-runtime
|
||||
#Multiline on
|
||||
#Multiline_Flush 5
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 5
|
||||
Path /etc/kubernetes/logs/containerd.log
|
||||
DB /etc/kubernetes/logs/gcp-containerd.log.pos
|
||||
|
||||
<match reform.**>
|
||||
@type record_reformer
|
||||
enable_ruby true
|
||||
<record>
|
||||
# Extract local_resource_id from tag for 'k8s_container' monitored
|
||||
# resource. The format is:
|
||||
# 'k8s_container.<namespace_name>.<pod_name>.<container_name>'.
|
||||
"logging.googleapis.com/local_resource_id" ${"k8s_container.#{tag_suffix[4].rpartition('.')[0].split('_')[1]}.#{tag_suffix[4].rpartition('.')[0].split('_')[0]}.#{tag_suffix[4].rpartition('.')[0].split('_')[2].rpartition('-')[0]}"}
|
||||
# Rename the field 'log' to a more generic field 'message'. This way the
|
||||
# fluent-plugin-google-cloud knows to flatten the field as textPayload
|
||||
# instead of jsonPayload after extracting 'time', 'severity' and
|
||||
# 'stream' from the record.
|
||||
message ${record['log']}
|
||||
# If 'severity' is not set, assume stderr is ERROR and stdout is INFO.
|
||||
severity ${record['severity'] || if record['stream'] == 'stderr' then 'ERROR' else 'INFO' end}
|
||||
</record>
|
||||
tag ${if record['stream'] == 'stderr' then 'raw.stderr' else 'raw.stdout' end}
|
||||
remove_keys stream,log
|
||||
</match>
|
||||
# Copied from fluentbit config. How is this used ? In match stages ?
|
||||
Parser_Firstline /^\w\d{4}/
|
||||
Parser_1 ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
|
||||
# TODO: detect exceptions and forward them as one log entry using the
|
||||
# detect_exceptions plugin
|
||||
# missing from fluentbit
|
||||
# time_format %m%d %H:%M:%S.%N
|
||||
|
||||
# This section is exclusive for k8s_container logs. These logs come with
|
||||
# 'raw.stderr' or 'raw.stdout' tags.
|
||||
<match {raw.stderr,raw.stdout}>
|
||||
@type google_cloud
|
||||
# Try to detect JSON formatted log entries.
|
||||
detect_json true
|
||||
# Allow log entries from multiple containers to be sent in the same request.
|
||||
split_logs_by_tag false
|
||||
# Set the buffer type to file to improve the reliability and reduce the memory consumption
|
||||
buffer_type file
|
||||
buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
|
||||
# Set queue_full action to block because we want to pause gracefully
|
||||
# in case of the off-the-limits load instead of throwing an exception
|
||||
buffer_queue_full_action block
|
||||
# Set the chunk limit conservatively to avoid exceeding the recommended
|
||||
# chunk size of 5MB per write request.
|
||||
buffer_chunk_limit 512k
|
||||
# Cap the combined memory usage of this buffer and the one below to
|
||||
# 512KiB/chunk * (6 + 2) chunks = 4 MiB
|
||||
buffer_queue_limit 6
|
||||
# Never wait more than 5 seconds before flushing logs in the non-error case.
|
||||
flush_interval 5s
|
||||
# Never wait longer than 30 seconds between retries.
|
||||
max_retry_wait 30
|
||||
# Disable the limit on the number of retries (retry forever).
|
||||
disable_retry_limit
|
||||
# Use multiple threads for processing.
|
||||
num_threads 2
|
||||
use_grpc true
|
||||
# Skip timestamp adjustment as this is in a controlled environment with
|
||||
# known timestamp format. This helps with CPU usage.
|
||||
adjust_invalid_timestamps false
|
||||
</match>
|
||||
|
||||
# Attach local_resource_id for 'k8s_node' monitored resource.
|
||||
<filter **>
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
<record>
|
||||
"logging.googleapis.com/local_resource_id" ${"k8s_node.NODE_NAME"}
|
||||
</record>
|
||||
</filter>
|
||||
# I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
|
||||
[INPUT]
|
||||
Name tail
|
||||
Alias kube-proxy
|
||||
Tag kube-proxy
|
||||
#Multiline on
|
||||
#Multiline_Flush 5
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 5
|
||||
Path /etc/kubernetes/logs/kube-proxy.log
|
||||
DB /etc/kubernetes/logs/gcp-kubeproxy.db
|
||||
|
||||
# Copied from fluentbit config. How is this used ? In match stages ?
|
||||
Parser_Firstline /^\w\d{4}/
|
||||
Parser_1 ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
|
||||
# missing from fluentbit
|
||||
# time_format %m%d %H:%M:%S.%N
|
||||
|
||||
[FILTER]
|
||||
Name modify
|
||||
Match *
|
||||
Hard_rename log message
|
||||
|
||||
# [OUTPUT]
|
||||
# Name http
|
||||
# Match *
|
||||
# Host 127.0.0.1
|
||||
# Port 2021
|
||||
# URI /logs
|
||||
# header_tag FLUENT-TAG
|
||||
# Format msgpack
|
||||
# Retry_Limit 2
|
||||
|
||||
[OUTPUT]
|
||||
name stackdriver
|
||||
match *
|
||||
'@
|
||||
|
||||
# Fluentbit parsers config file
|
||||
$PARSERS_CONFIG = @'
|
||||
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
||||
|
||||
[PARSER]
|
||||
Name containerd
|
||||
Format regex
|
||||
Regex ^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
||||
|
||||
[PARSER]
|
||||
Name json
|
||||
Format json
|
||||
|
||||
[PARSER]
|
||||
Name syslog
|
||||
Format regex
|
||||
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
|
||||
Time_Key time
|
||||
Time_Format %b %d %H:%M:%S
|
||||
|
||||
[PARSER]
|
||||
Name glog
|
||||
Format regex
|
||||
Regex ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source_file>[^ \]]+)\:(?<source_line>\d+)\]\s(?<message>.*)$
|
||||
Time_Key time
|
||||
Time_Format %m%d %H:%M:%S.%L
|
||||
|
||||
[PARSER]
|
||||
Name network-log
|
||||
Format json
|
||||
Time_Key timestamp
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
||||
|
||||
# ----------
|
||||
|
||||
[PARSER]
|
||||
Name json
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %d/%b/%Y:%H:%M:%S %z
|
||||
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
||||
Time_Keep On
|
||||
|
||||
|
||||
[PARSER]
|
||||
Name syslog-rfc5424
|
||||
Format regex
|
||||
Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
||||
Time_Keep On
|
||||
|
||||
[PARSER]
|
||||
Name syslog-rfc3164-local
|
||||
Format regex
|
||||
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
|
||||
Time_Key time
|
||||
Time_Format %b %d %H:%M:%S
|
||||
Time_Keep On
|
||||
|
||||
[PARSER]
|
||||
Name syslog-rfc3164
|
||||
Format regex
|
||||
Regex /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
||||
Time_Key time
|
||||
Time_Format %b %d %H:%M:%S
|
||||
Time_Keep On
|
||||
|
||||
[PARSER]
|
||||
Name kube-custom
|
||||
Format regex
|
||||
Regex (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
|
||||
'@
|
||||
|
||||
# Export all public functions:
|
||||
Export-ModuleMember -Function *-*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user