github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-09 03:57:41 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #112957 from mxpv/log-dir

Browse Source 
Allow changing pod log directory
This commit is contained in:
Kubernetes Prow Robot 2024-03-04 21:07:06 -08:00 committed by GitHub
commit dc3f5ec6cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 185 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kubelet/app/server.go
View File

@ -77,7 +77,7 @@ import (
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
tracing "k8s.io/component-base/tracing"
"k8s.io/component-base/tracing"
"k8s.io/component-base/version"
"k8s.io/component-base/version/verflag"
nodeutil "k8s.io/component-helpers/node/util"
@ -1292,6 +1292,7 @@ func createAndInitKubelet(kubeServer *options.KubeletServer,
kubeServer.CloudProvider,
kubeServer.CertDirectory,
kubeServer.RootDirectory,
kubeServer.PodLogsDir,
kubeServer.ImageCredentialProviderConfigFile,
kubeServer.ImageCredentialProviderBinDir,
kubeServer.RegisterNode,

7
pkg/generated/openapi/zz_generated.openapi.go generated
View File

@ -58613,6 +58613,13 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen
Format: "",
},
},
"podLogsDir": {
SchemaProps: spec.SchemaProps{
Description: "podLogsDir is a custom root directory path kubelet will use to place pod's log files. Default: \"/var/log/pods/\" Note: it is not recommended to use the temp folder as a log directory as it may cause unexpected behavior in many places.",
Type: []string{"string"},
Format: "",
},
},
"syncFrequency": {
SchemaProps: spec.SchemaProps{
Description: "syncFrequency is the max period between synchronizing running containers and config. Default: \"1m\"",

1
pkg/kubelet/apis/config/fuzzer/fuzzer.go
View File

@ -83,6 +83,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
"memory": "50%",
}
obj.OOMScoreAdj = int32(qos.KubeletOOMScoreAdj)
obj.PodLogsDir = "/var/log/pods"
obj.Port = ports.KubeletPort
obj.ReadOnlyPort = ports.KubeletReadOnlyPort
obj.RegistryBurst = 10

1
pkg/kubelet/apis/config/helpers.go
View File

@ -27,5 +27,6 @@ func KubeletConfigurationPathRefs(kc *KubeletConfiguration) []*string {
paths = append(paths, &kc.TLSPrivateKeyFile)
paths = append(paths, &kc.ResolverConfig)
paths = append(paths, &kc.VolumePluginDir)
paths = append(paths, &kc.PodLogsDir)
return paths
}

1
pkg/kubelet/apis/config/helpers_test.go
View File

@ -156,6 +156,7 @@ var (
"TLSCertFile",
"TLSPrivateKeyFile",
"ResolverConfig",
"PodLogsDir",
)
// KubeletConfiguration fields that do not contain file paths.

1
pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml vendored
View File

@ -73,6 +73,7 @@ nodeStatusMaxImages: 50
nodeStatusReportFrequency: 5m0s
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podLogsDir: /var/log/pods
podPidsLimit: -1
port: 10250
registerNode: true

1
pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml vendored
View File

@ -73,6 +73,7 @@ nodeStatusMaxImages: 50
nodeStatusReportFrequency: 5m0s
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podLogsDir: /var/log/pods
podPidsLimit: -1
port: 10250
registerNode: true

5
pkg/kubelet/apis/config/types.go
View File

@ -88,6 +88,11 @@ type KubeletConfiguration struct {
// staticPodPath is the path to the directory containing local (static) pods to
// run, or the path to a single static pod file.
StaticPodPath string
// podLogsDir is a custom root directory path kubelet will use to place pod's log files.
// Default: "/var/log/pods/"
// Note: it is not recommended to use the temp folder as a log directory as it may cause
// unexpected behavior in many places.
PodLogsDir string
// syncFrequency is the max period between synchronizing running
// containers and config
SyncFrequency metav1.Duration

5
pkg/kubelet/apis/config/v1beta1/defaults.go
View File

@ -36,7 +36,7 @@ const (
DefaultIPTablesMasqueradeBit = 14
DefaultIPTablesDropBit = 15
DefaultVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
DefaultPodLogsDir = "/var/log/pods"
// See https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2570-memory-qos
DefaultMemoryThrottlingFactor = 0.9
)
@ -280,4 +280,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
if obj.ContainerRuntimeEndpoint == "" {
obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"
}
if obj.PodLogsDir == "" {
obj.PodLogsDir = DefaultPodLogsDir
}
}

8
pkg/kubelet/apis/config/v1beta1/defaults_test.go
View File

@ -128,6 +128,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(DefaultMemoryThrottlingFactor),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: DefaultPodLogsDir,
},
},
{
@ -257,6 +258,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(0),
RegisterNode: utilpointer.Bool(false),
LocalStorageCapacityIsolation: utilpointer.Bool(false),
PodLogsDir: "",
},
&v1beta1.KubeletConfiguration{
EnableServer: utilpointer.Bool(false),
@ -357,6 +359,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(0),
RegisterNode: utilpointer.Bool(false),
LocalStorageCapacityIsolation: utilpointer.Bool(false),
PodLogsDir: DefaultPodLogsDir,
},
},
{
@ -508,6 +511,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(1),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: "/custom/path",
},
&v1beta1.KubeletConfiguration{
EnableServer: utilpointer.Bool(true),
@ -656,6 +660,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(1),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: "/custom/path",
},
},
{
@ -747,6 +752,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: DefaultPodLogsDir,
},
},
{
@ -838,6 +844,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: DefaultPodLogsDir,
},
},
{
@ -929,6 +936,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemoryThrottlingFactor: utilpointer.Float64(DefaultMemoryThrottlingFactor),
RegisterNode: utilpointer.Bool(true),
LocalStorageCapacityIsolation: utilpointer.Bool(true),
PodLogsDir: DefaultPodLogsDir,
},
},
}

2
pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go generated
View File

@ -346,6 +346,7 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
return err
}
out.StaticPodPath = in.StaticPodPath
out.PodLogsDir = in.PodLogsDir
out.SyncFrequency = in.SyncFrequency
out.FileCheckFrequency = in.FileCheckFrequency
out.HTTPCheckFrequency = in.HTTPCheckFrequency
@ -538,6 +539,7 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
return err
}
out.StaticPodPath = in.StaticPodPath
out.PodLogsDir = in.PodLogsDir
out.SyncFrequency = in.SyncFrequency
out.FileCheckFrequency = in.FileCheckFrequency
out.HTTPCheckFrequency = in.HTTPCheckFrequency

23
pkg/kubelet/apis/config/validation/validation.go
View File

@ -18,7 +18,9 @@ package validation
import (
"fmt"
"path/filepath"
"time"
"unicode"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
@ -286,5 +288,26 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
if kc.ContainerLogMonitorInterval.Duration.Seconds() < 3 {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: containerLogMonitorInterval must be a positive time duration greater than or equal to 3s"))
}
if kc.PodLogsDir == "" {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: podLogsDir was not specified"))
}
if !filepath.IsAbs(kc.PodLogsDir) {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: pod logs path %q must be absolute path", kc.PodLogsDir))
}
if filepath.Clean(kc.PodLogsDir) != kc.PodLogsDir {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: pod logs path %q must be normalized", kc.PodLogsDir))
}
// Since pod logs path is used in metrics, make sure it contains only ASCII characters.
for _, c := range kc.PodLogsDir {
if c > unicode.MaxASCII {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: pod logs path %q mut contains ASCII characters only", kc.PodLogsDir))
break
}
}
return utilerrors.NewAggregate(allErrors)
}

32
pkg/kubelet/apis/config/validation/validation_test.go
View File

@ -38,6 +38,7 @@ var (
EnforceNodeAllocatable: enforceNodeAllocatable,
SystemReservedCgroup: "/system.slice",
KubeReservedCgroup: "/kubelet.service",
PodLogsDir: "/logs",
SystemCgroups: "",
CgroupRoot: "",
EventBurst: 10,
@ -569,7 +570,36 @@ func TestValidateKubeletConfiguration(t *testing.T) {
return conf
},
errMsg: "invalid configuration: containerLogMonitorInterval must be a positive time duration greater than or equal to 3s",
}}
}, {
name: "pod logs path must be not empty",
configure: func(config *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
config.PodLogsDir = ""
return config
},
errMsg: "invalid configuration: podLogsDir was not specified",
}, {
name: "pod logs path must be absolute",
configure: func(config *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
config.PodLogsDir = "./test"
return config
},
errMsg: `invalid configuration: pod logs path "./test" must be absolute path`,
}, {
name: "pod logs path must be normalized",
configure: func(config *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
config.PodLogsDir = "/path/../"
return config
},
errMsg: `invalid configuration: pod logs path "/path/../" must be normalized`,
}, {
name: "pod logs path is ascii only",
configure: func(config *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
config.PodLogsDir = "/🧪"
return config
},
errMsg: `invalid configuration: pod logs path "/🧪" mut contains ASCII characters only`,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {

14
pkg/kubelet/kubelet.go
View File

@ -19,6 +19,7 @@ package kubelet
import (
"context"
"crypto/tls"
"errors"
"fmt"
"math"
"net"
@ -346,6 +347,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
cloudProvider string,
certDirectory string,
rootDirectory string,
podLogsDirectory string,
imageCredentialProviderConfigFile string,
imageCredentialProviderBinDir string,
registerNode bool,
@ -369,6 +371,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
if rootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
}
if podLogsDirectory == "" {
return nil, errors.New("pod logs root directory is empty")
}
if kubeCfg.SyncFrequency.Duration <= 0 {
return nil, fmt.Errorf("invalid sync frequency %d", kubeCfg.SyncFrequency.Duration)
}
@ -518,6 +523,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
heartbeatClient: kubeDeps.HeartbeatClient,
onRepeatedHeartbeatFailure: kubeDeps.OnHeartbeatFailure,
rootDirectory: filepath.Clean(rootDirectory),
podLogsDirectory: podLogsDirectory,
resyncInterval: kubeCfg.SyncFrequency.Duration,
sourcesReady: config.NewSourcesReady(kubeDeps.PodConfig.SeenAllSources),
registerNode: registerNode,
@ -648,6 +654,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.readinessManager,
klet.startupManager,
rootDirectory,
podLogsDirectory,
machineInfo,
klet.podWorkers,
kubeDeps.OSInterface,
@ -690,7 +697,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// common provider to get host file system usage associated with a pod managed by kubelet
hostStatsProvider := stats.NewHostStatsProvider(kubecontainer.RealOS{}, func(podUID types.UID) string {
return getEtcHostsPath(klet.getPodDir(podUID))
})
}, podLogsDirectory)
if kubeDeps.useLegacyCadvisorStats {
klet.StatsProvider = stats.NewCadvisorStatsProvider(
klet.cadvisor,
@ -953,6 +960,7 @@ type Kubelet struct {
mirrorPodClient kubepod.MirrorClient
rootDirectory string
podLogsDirectory string
lastObservedNodeAddressesMux sync.RWMutex
lastObservedNodeAddresses []v1.NodeAddress
@ -1372,6 +1380,7 @@ func (kl *Kubelet) RlimitStats() (*statsapi.RlimitStats, error) {
// 3. the plugins directory
// 4. the pod-resources directory
// 5. the checkpoint directory
// 6. the pod logs root directory
func (kl *Kubelet) setupDataDirs() error {
if cleanedRoot := filepath.Clean(kl.rootDirectory); cleanedRoot != kl.rootDirectory {
return fmt.Errorf("rootDirectory not in canonical form: expected %s, was %s", cleanedRoot, kl.rootDirectory)
@ -1381,6 +1390,9 @@ func (kl *Kubelet) setupDataDirs() error {
if err := os.MkdirAll(kl.getRootDir(), 0750); err != nil {
return fmt.Errorf("error creating root directory: %v", err)
}
if err := os.MkdirAll(kl.getPodLogsDir(), 0750); err != nil {
return fmt.Errorf("error creating pod logs root directory %q: %w", kl.getPodLogsDir(), err)
}
if err := kl.hostutil.MakeRShared(kl.getRootDir()); err != nil {
return fmt.Errorf("error configuring root directory: %v", err)
}

7
pkg/kubelet/kubelet_getters.go
View File

@ -48,6 +48,13 @@ func (kl *Kubelet) getRootDir() string {
return kl.rootDirectory
}
// getPodLogsDir returns the full path to the directory that kubelet can use
// to store pod's log files. This defaults to /var/log/pods if not specified
// otherwise in the config file.
func (kl *Kubelet) getPodLogsDir() string {
return kl.podLogsDirectory
}
// getPodsDir returns the full path to the directory under which pod
// directories are created.
func (kl *Kubelet) getPodsDir() string {

3
pkg/kubelet/kubelet_getters_test.go
View File

@ -36,6 +36,9 @@ func TestKubeletDirs(t *testing.T) {
exp = filepath.Join(root, "pods")
assert.Equal(t, exp, got)
got = kubelet.getPodLogsDir()
assert.Equal(t, kubelet.podLogsDirectory, got)
got = kubelet.getPluginsDir()
exp = filepath.Join(root, "plugins")
assert.Equal(t, exp, got)

12
pkg/kubelet/kubelet_test.go
View File

@ -206,14 +206,8 @@ func newTestKubeletWithImageList(
kubelet.nodeName = types.NodeName(testKubeletHostname)
kubelet.runtimeState = newRuntimeState(maxWaitForContainerRuntime)
kubelet.runtimeState.setNetworkState(nil)
if tempDir, err := os.MkdirTemp("", "kubelet_test."); err != nil {
t.Fatalf("can't make a temp rootdir: %v", err)
} else {
kubelet.rootDirectory = tempDir
}
if err := os.MkdirAll(kubelet.rootDirectory, 0750); err != nil {
t.Fatalf("can't mkdir(%q): %v", kubelet.rootDirectory, err)
}
kubelet.rootDirectory = t.TempDir()
kubelet.podLogsDirectory = t.TempDir()
kubelet.sourcesReady = config.NewSourcesReady(func(_ sets.String) bool { return true })
kubelet.serviceLister = testServiceLister{}
kubelet.serviceHasSynced = func() bool { return true }
@ -3112,6 +3106,7 @@ func TestNewMainKubeletStandAlone(t *testing.T) {
"external",
"/tmp/cert",
"/tmp/rootdir",
tempDir,
"",
"",
false,
@ -3206,6 +3201,7 @@ func TestSyncPodSpans(t *testing.T) {
kubelet.readinessManager,
kubelet.startupManager,
kubelet.rootDirectory,
kubelet.podLogsDirectory,
kubelet.machineInfo,
kubelet.podWorkers,
kubelet.os,

3
pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go
View File

@ -46,6 +46,8 @@ const (
fakeNodeAllocatableMemory = "32Gi"
fakeNodeAllocatableCPU = "16"
fakePodLogsDirectory = "/var/log/pods"
)
type fakeHTTP struct {
@ -115,6 +117,7 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
logReduction: logreduction.NewLogReduction(identicalErrorDelay),
logManager: logManager,
memoryThrottlingFactor: 0.9,
podLogsDirectory: fakePodLogsDirectory,
}
typedVersion, err := runtimeService.Version(ctx, kubeRuntimeAPIVersion)

8
pkg/kubelet/kuberuntime/helpers.go
View File

@ -188,13 +188,13 @@ func buildContainerLogsPath(containerName string, restartCount int) string {
}
// BuildContainerLogsDirectory builds absolute log directory path for a container in pod.
func BuildContainerLogsDirectory(podNamespace, podName string, podUID types.UID, containerName string) string {
return filepath.Join(BuildPodLogsDirectory(podNamespace, podName, podUID), containerName)
func BuildContainerLogsDirectory(podLogsDir, podNamespace, podName string, podUID types.UID, containerName string) string {
return filepath.Join(BuildPodLogsDirectory(podLogsDir, podNamespace, podName, podUID), containerName)
}
// BuildPodLogsDirectory builds absolute log directory path for a pod sandbox.
func BuildPodLogsDirectory(podNamespace, podName string, podUID types.UID) string {
return filepath.Join(podLogsRootDirectory, strings.Join([]string{podNamespace, podName,
func BuildPodLogsDirectory(podLogsDir, podNamespace, podName string, podUID types.UID) string {
return filepath.Join(podLogsDir, strings.Join([]string{podNamespace, podName,
string(podUID)}, logPathDelimiter))
}

4
pkg/kubelet/kuberuntime/kuberuntime_container.go
View File

@ -219,7 +219,7 @@ func (m *kubeGenericRuntimeManager) startContainer(ctx context.Context, podSandb
// We are checking to see if the log directory exists, and find
// the latest restartCount by checking the log name -
// {restartCount}.log - and adding 1 to it.
logDir := BuildContainerLogsDirectory(pod.Namespace, pod.Name, pod.UID, container.Name)
logDir := BuildContainerLogsDirectory(m.podLogsDirectory, pod.Namespace, pod.Name, pod.UID, container.Name)
restartCount, err = calcRestartCountByLogDir(logDir)
if err != nil {
klog.InfoS("Cannot calculate restartCount from the log directory", "logDir", logDir, "err", err)
@ -334,7 +334,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(ctx context.Context,
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
logDir := BuildContainerLogsDirectory(pod.Namespace, pod.Name, pod.UID, container.Name)
logDir := BuildContainerLogsDirectory(m.podLogsDirectory, pod.Namespace, pod.Name, pod.UID, container.Name)
err = m.osInterface.MkdirAll(logDir, 0755)
if err != nil {
return nil, cleanupAction, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)

5
pkg/kubelet/kuberuntime/kuberuntime_container_test.go
View File

@ -78,8 +78,9 @@ func TestRemoveContainer(t *testing.T) {
pattern = strings.Replace(pattern, "\\", "\\\\", -1)
return regexp.MustCompile(pattern).MatchString(path)
}
expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "new_bar_12345678", "foo", "0.log")
expectedContainerLogPathRotated := filepath.Join(podLogsRootDirectory, "new_bar_12345678", "foo", "0.log.20060102-150405")
podLogsDirectory := "/var/log/pods"
expectedContainerLogPath := filepath.Join(podLogsDirectory, "new_bar_12345678", "foo", "0.log")
expectedContainerLogPathRotated := filepath.Join(podLogsDirectory, "new_bar_12345678", "foo", "0.log.20060102-150405")
expectedContainerLogSymlink := legacyLogSymlink(containerID, "foo", "bar", "new")
fakeOS.Create(expectedContainerLogPath)

7
pkg/kubelet/kuberuntime/kuberuntime_gc.go
View File

@ -327,11 +327,12 @@ func (cgc *containerGC) evictSandboxes(ctx context.Context, evictNonDeletedPods
// are evictable if there are no corresponding pods.
func (cgc *containerGC) evictPodLogsDirectories(ctx context.Context, allSourcesReady bool) error {
osInterface := cgc.manager.osInterface
podLogsDirectory := cgc.manager.podLogsDirectory
if allSourcesReady {
// Only remove pod logs directories when all sources are ready.
dirs, err := osInterface.ReadDir(podLogsRootDirectory)
dirs, err := osInterface.ReadDir(podLogsDirectory)
if err != nil {
return fmt.Errorf("failed to read podLogsRootDirectory %q: %v", podLogsRootDirectory, err)
return fmt.Errorf("failed to read podLogsDirectory %q: %w", podLogsDirectory, err)
}
for _, dir := range dirs {
name := dir.Name()
@ -340,7 +341,7 @@ func (cgc *containerGC) evictPodLogsDirectories(ctx context.Context, allSourcesR
continue
}
klog.V(4).InfoS("Removing pod logs", "podUID", podUID)
err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name))
err := osInterface.RemoveAll(filepath.Join(podLogsDirectory, name))
if err != nil {
klog.ErrorS(err, "Failed to remove pod logs directory", "path", name)
}

7
pkg/kubelet/kuberuntime/kuberuntime_gc_test.go
View File

@ -430,10 +430,11 @@ func TestPodLogDirectoryGC(t *testing.T) {
// pod log directories without corresponding pods should be removed.
files := []string{"123", "456", "789", "012", "name_namespace_321", "name_namespace_654"}
podLogsDirectory := "/var/log/pods"
removed := []string{
filepath.Join(podLogsRootDirectory, "789"),
filepath.Join(podLogsRootDirectory, "012"),
filepath.Join(podLogsRootDirectory, "name_namespace_654"),
filepath.Join(podLogsDirectory, "789"),
filepath.Join(podLogsDirectory, "012"),
filepath.Join(podLogsDirectory, "name_namespace_654"),
}
podStateProvider.removed["012"] = struct{}{}
podStateProvider.removed["789"] = struct{}{}

16
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@ -70,8 +70,6 @@ import (
const (
// The api version of kubelet runtime api
kubeRuntimeAPIVersion = "0.1.0"
// The root directory for pod logs
podLogsRootDirectory = "/var/log/pods"
// A minimal shutdown window for avoiding unnecessary SIGKILLs
minimumGracePeriodInSeconds = 2
@ -169,6 +167,9 @@ type kubeGenericRuntimeManager struct {
// Memory throttling factor for MemoryQoS
memoryThrottlingFactor float64
// Root directory used to store pod logs
podLogsDirectory string
}
// KubeGenericRuntime is a interface contains interfaces for container runtime and command.
@ -185,6 +186,7 @@ func NewKubeGenericRuntimeManager(
readinessManager proberesults.Manager,
startupManager proberesults.Manager,
rootDirectory string,
podLogsDirectory string,
machineInfo *cadvisorapi.MachineInfo,
podStateProvider podStateProvider,
osInterface kubecontainer.OSInterface,
@ -237,6 +239,7 @@ func NewKubeGenericRuntimeManager(
memorySwapBehavior: memorySwapBehavior,
getNodeAllocatable: getNodeAllocatable,
memoryThrottlingFactor: memoryThrottlingFactor,
podLogsDirectory: podLogsDirectory,
}
typedVersion, err := kubeRuntimeManager.getTypedVersion(ctx)
@ -260,15 +263,6 @@ func NewKubeGenericRuntimeManager(
"version", typedVersion.RuntimeVersion,
"apiVersion", typedVersion.RuntimeApiVersion)
// If the container logs directory does not exist, create it.
// TODO: create podLogsRootDirectory at kubelet.go when kubelet is refactored to
// new runtime interface
if _, err := osInterface.Stat(podLogsRootDirectory); os.IsNotExist(err) {
if err := osInterface.MkdirAll(podLogsRootDirectory, 0755); err != nil {
klog.ErrorS(err, "Failed to create pod log directory", "path", podLogsRootDirectory)
}
}
if imageCredentialProviderConfigFile != "" || imageCredentialProviderBinDir != "" {
if err := plugin.RegisterCredentialProviderPlugins(imageCredentialProviderConfigFile, imageCredentialProviderBinDir); err != nil {
klog.ErrorS(err, "Failed to register CRI auth plugins")

2
pkg/kubelet/kuberuntime/kuberuntime_sandbox.go
View File

@ -111,7 +111,7 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp
podSandboxConfig.Hostname = podHostname
}
logDir := BuildPodLogsDirectory(pod.Namespace, pod.Name, pod.UID)
logDir := BuildPodLogsDirectory(m.podLogsDirectory, pod.Namespace, pod.Name, pod.UID)
podSandboxConfig.LogDirectory = logDir
portMappings := []*runtimeapi.PortMapping{}

6
pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go
View File

@ -37,12 +37,14 @@ import (
"k8s.io/utils/pointer"
)
const testPodLogsDirectory = "/var/log/pods"
func TestGeneratePodSandboxConfig(t *testing.T) {
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
pod := newTestPod()
expectedLogDirectory := filepath.Join(podLogsRootDirectory, pod.Namespace+"_"+pod.Name+"_12345678")
expectedLogDirectory := filepath.Join(testPodLogsDirectory, pod.Namespace+"_"+pod.Name+"_12345678")
expectedLabels := map[string]string{
"io.kubernetes.pod.name": pod.Name,
"io.kubernetes.pod.namespace": pod.Namespace,
@ -78,7 +80,7 @@ func TestCreatePodSandbox(t *testing.T) {
fakeOS := m.osInterface.(*containertest.FakeOS)
fakeOS.MkdirAllFn = func(path string, perm os.FileMode) error {
// Check pod logs root directory is created.
assert.Equal(t, filepath.Join(podLogsRootDirectory, pod.Namespace+"_"+pod.Name+"_12345678"), path)
assert.Equal(t, filepath.Join(testPodLogsDirectory, pod.Namespace+"_"+pod.Name+"_12345678"), path)
assert.Equal(t, os.FileMode(0755), perm)
return nil
}

1
pkg/kubelet/runonce_test.go
View File

@ -84,6 +84,7 @@ func TestRunOnce(t *testing.T) {
defer os.RemoveAll(basePath)
kb := &Kubelet{
rootDirectory: filepath.Clean(basePath),
podLogsDirectory: filepath.Join(basePath, "pod-logs"),
recorder: &record.FakeRecorder{},
cadvisor: cadvisor,
nodeLister: testNodeLister{},

4
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -706,8 +706,8 @@ func TestCadvisorListPodStatsWhenContainerLogFound(t *testing.T) {
containerLogStats0 := makeFakeLogStats(0)
containerLogStats1 := makeFakeLogStats(0)
fakeStats := map[string]*volume.Metrics{
kuberuntime.BuildContainerLogsDirectory(prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName00): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory(prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName01): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName00): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName01): containerLogStats1,
}
fakeStatsSlice := []*volume.Metrics{containerLogStats0, containerLogStats1}
fakeOS := &containertest.FakeOS{}

38
pkg/kubelet/stats/cri_stats_provider_test.go
View File

@ -26,7 +26,7 @@ import (
"testing"
"time"
gomock "github.com/golang/mock/gomock"
"github.com/golang/mock/gomock"
cadvisorfs "github.com/google/cadvisor/fs"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
@ -89,6 +89,8 @@ const (
cName9 = "container9-name"
)
const testPodLogDirectory = "/var/log/kube/pods/" // Use non-default path to ensure stats are collected properly
func TestCRIListPodStats(t *testing.T) {
ctx := context.Background()
var (
@ -204,14 +206,14 @@ func TestCRIListPodStats(t *testing.T) {
}
fakeStats := map[string]*volume.Metrics{
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
kuberuntime.BuildContainerLogsDirectory("sandbox2-ns", "sandbox2-name", types.UID("sandbox2-uid"), cName3): containerLogStats4,
kuberuntime.BuildContainerLogsDirectory("sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName5): containerLogStats5,
kuberuntime.BuildContainerLogsDirectory("sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName8): containerLogStats8,
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox2-ns", "sandbox2-name", types.UID("sandbox2-uid"), cName3): containerLogStats4,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName5): containerLogStats5,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName8): containerLogStats8,
filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
}
ctrl := gomock.NewController(t)
@ -222,9 +224,9 @@ func TestCRIListPodStats(t *testing.T) {
var dirEntries []os.DirEntry
mockDE := kubecontainertest.NewMockDirEntry(ctrl)
switch path {
case kuberuntime.BuildPodLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
mockDE.EXPECT().Name().Return(podLogName0)
case kuberuntime.BuildPodLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
mockDE.EXPECT().Name().Return(podLogName1)
default:
return nil, nil
@ -431,11 +433,11 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
PersistentVolumes: persistentVolumes,
}
fakeStats := map[string]*volume.Metrics{
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@ -444,9 +446,9 @@ func TestListPodStatsStrictlyFromCRI(t *testing.T) {
var dirEntries []os.DirEntry
mockDE := kubecontainertest.NewMockDirEntry(ctrl)
switch path {
case kuberuntime.BuildPodLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
mockDE.EXPECT().Name().Return(podLogName0)
case kuberuntime.BuildPodLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
mockDE.EXPECT().Name().Return(podLogName1)
default:
return nil, nil

2
pkg/kubelet/stats/cri_stats_provider_windows_test.go
View File

@ -455,7 +455,7 @@ func Test_criStatsProvider_makeWinContainerStats(t *testing.T) {
InodesUsed: resource.NewQuantity(int64(logStatsInodesUsed), resource.BinarySI),
}
fakeStats := map[string]*volume.Metrics{
kuberuntime.BuildContainerLogsDirectory("sb0-ns", "sb0-name", types.UID("sb0-uid"), "c0"): c0LogStats,
kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sb0-ns", "sb0-name", types.UID("sb0-uid"), "c0"): c0LogStats,
}
fakeOS := &kubecontainertest.FakeOS{}
fakeHostStatsProvider := NewFakeHostStatsProviderWithData(fakeStats, fakeOS)

9
pkg/kubelet/stats/host_stats_provider.go
View File

@ -51,13 +51,16 @@ type hostStatsProvider struct {
osInterface kubecontainer.OSInterface
// podEtcHostsPathFunc fetches a pod etc hosts path by uid.
podEtcHostsPathFunc PodEtcHostsPathFunc
// podLogsDirectory is the root directory path for pod logs.
podLogsDirectory string
}
// NewHostStatsProvider returns a new HostStatsProvider type struct.
func NewHostStatsProvider(osInterface kubecontainer.OSInterface, podEtcHostsPathFunc PodEtcHostsPathFunc) HostStatsProvider {
func NewHostStatsProvider(osInterface kubecontainer.OSInterface, podEtcHostsPathFunc PodEtcHostsPathFunc, podLogsDirectory string) HostStatsProvider {
return hostStatsProvider{
osInterface: osInterface,
podEtcHostsPathFunc: podEtcHostsPathFunc,
podLogsDirectory: podLogsDirectory,
}
}
@ -102,12 +105,12 @@ func (h hostStatsProvider) getPodEtcHostsStats(podUID types.UID, rootFsInfo *cad
}
func (h hostStatsProvider) podLogMetrics(podNamespace, podName string, podUID types.UID) (metricsProviderByPath, error) {
podLogsDirectoryPath := kuberuntime.BuildPodLogsDirectory(podNamespace, podName, podUID)
podLogsDirectoryPath := kuberuntime.BuildPodLogsDirectory(h.podLogsDirectory, podNamespace, podName, podUID)
return h.fileMetricsByDir(podLogsDirectoryPath)
}
func (h hostStatsProvider) podContainerLogMetrics(podNamespace, podName string, podUID types.UID, containerName string) (metricsProviderByPath, error) {
podContainerLogsDirectoryPath := kuberuntime.BuildContainerLogsDirectory(podNamespace, podName, podUID, containerName)
podContainerLogsDirectoryPath := kuberuntime.BuildContainerLogsDirectory(h.podLogsDirectory, podNamespace, podName, podUID, containerName)
return h.fileMetricsByDir(podContainerLogsDirectoryPath)
}

4
pkg/kubelet/stats/host_stats_provider_fake.go
View File

@ -50,7 +50,7 @@ func NewFakeHostStatsProviderWithData(fakeStats map[string]*volume.Metrics, osIn
}
func (f *fakeHostStatsProvider) getPodLogStats(podNamespace, podName string, podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
path := kuberuntime.BuildPodLogsDirectory(podNamespace, podName, podUID)
path := kuberuntime.BuildPodLogsDirectory("/var/log/kube/pods/", podNamespace, podName, podUID)
files, err := f.osInterface.ReadDir(path)
if err != nil {
return nil, err
@ -68,7 +68,7 @@ func (f *fakeHostStatsProvider) getPodLogStats(podNamespace, podName string, pod
}
func (f *fakeHostStatsProvider) getPodContainerLogStats(podNamespace, podName string, podUID types.UID, containerName string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
path := kuberuntime.BuildContainerLogsDirectory(podNamespace, podName, podUID, containerName)
path := kuberuntime.BuildContainerLogsDirectory("/var/log/kube/pods/", podNamespace, podName, podUID, containerName)
metricsProvider := NewFakeMetricsDu(path, f.fakeStats[path])
return fakeMetricsProvidersToStats([]volume.MetricsProvider{metricsProvider}, rootFsInfo)
}

2
pkg/kubemark/hollow_kubelet.go
View File

@ -152,6 +152,7 @@ type HollowKubeletOptions struct {
func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
podLogsPath := utils.MakeTempDirOrDie("pod-logs", testRootDir)
klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
// Flags struct
@ -210,6 +211,7 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *
c.RegisterWithTaints = opt.RegisterWithTaints
c.RegisterNode = true
c.LocalStorageCapacityIsolation = true
c.PodLogsDir = podLogsPath
return f, c
}

6
staging/src/k8s.io/kubelet/config/v1beta1/types.go
View File

@ -98,6 +98,12 @@ type KubeletConfiguration struct {
// Default: ""
// +optional
StaticPodPath string `json:"staticPodPath,omitempty"`
// podLogsDir is a custom root directory path kubelet will use to place pod's log files.
// Default: "/var/log/pods/"
// Note: it is not recommended to use the temp folder as a log directory as it may cause
// unexpected behavior in many places.
// +optional
PodLogsDir string `json:"podLogsDir,omitempty"`
// syncFrequency is the max period between synchronizing running
// containers and config.
// Default: "1m"