From 3e9b24cffb34b1d59a6945ba48877e34acaef1d1 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 5 Jan 2026 16:19:01 +0000 Subject: [PATCH 1/2] fix(kubelet): convert V().Error() to V().Info() for verbosity-aware logging The go-logr package ignores verbosity level (V()) for Error() calls, causing these logs to always print regardless of the -v flag setting. Convert to Info() with "err" key-value pair to respect verbosity levels. This change covers 23 instances across kubelet components: - prober (4 instances) - lifecycle (4 instances) - stats (3 instances) - kuberuntime (2 instances) - pleg (1 instance) - allocation (1 instance) - cm/devicemanager (1 instance) - cm/dra (1 instance) - metrics/collectors (1 instance) - kubelet.go (3 instances) - images/pullmanager (2 instances) Ref: https://github.com/kubernetes/kubernetes/issues/136027 Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- pkg/kubelet/allocation/allocation_manager.go | 2 +- pkg/kubelet/cm/devicemanager/plugin/v1beta1/client.go | 2 +- pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go | 2 +- pkg/kubelet/kuberuntime/kuberuntime_container_linux.go | 4 ++-- pkg/kubelet/lifecycle/handlers.go | 8 ++++---- pkg/kubelet/metrics/collectors/cri_metrics.go | 2 +- pkg/kubelet/pleg/generic.go | 2 +- pkg/kubelet/prober/prober.go | 2 +- pkg/kubelet/prober/prober_manager.go | 6 +++--- pkg/kubelet/stats/cri_stats_provider.go | 4 +--- pkg/kubelet/stats/helper.go | 4 ++-- 11 files changed, 18 insertions(+), 20 deletions(-) diff --git a/pkg/kubelet/allocation/allocation_manager.go b/pkg/kubelet/allocation/allocation_manager.go index ef9edc64191..9a1414f1507 100644 --- a/pkg/kubelet/allocation/allocation_manager.go +++ b/pkg/kubelet/allocation/allocation_manager.go @@ -573,7 +573,7 @@ func (m *manager) RemovePod(uid types.UID) { logger := klog.TODO() if err := m.allocated.RemovePod(uid); err != nil { // If the deletion fails, it will be retried by RemoveOrphanedPods, so we can safely ignore the error. - logger.V(3).Error(err, "Failed to delete pod allocation", "podUID", uid) + logger.V(3).Info("Failed to delete pod allocation", "podUID", uid, "err", err) } } diff --git a/pkg/kubelet/cm/devicemanager/plugin/v1beta1/client.go b/pkg/kubelet/cm/devicemanager/plugin/v1beta1/client.go index 751ebc17ebb..a6e0e79c10f 100644 --- a/pkg/kubelet/cm/devicemanager/plugin/v1beta1/client.go +++ b/pkg/kubelet/cm/devicemanager/plugin/v1beta1/client.go @@ -104,7 +104,7 @@ func (c *client) Disconnect(logger klog.Logger) error { c.mutex.Lock() if c.grpc != nil { if err := c.grpc.Close(); err != nil { - logger.V(2).Error(err, "Failed to close grpc connection", "resource", c.Resource()) + logger.V(2).Info("Failed to close grpc connection", "resource", c.Resource(), "err", err) } c.grpc = nil } diff --git a/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go b/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go index 5305546ccf9..fd08372c93b 100644 --- a/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go +++ b/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go @@ -395,7 +395,7 @@ func (pm *DRAPluginManager) add(driverName string, endpoint string, chosenServic logger.V(4).Info("Attempting to start WatchResources health stream") stream, err := p.NodeWatchResources(ctx) if err != nil { - logger.V(3).Error(err, "Failed to establish WatchResources stream, will retry") + logger.V(3).Info("Failed to establish WatchResources stream, will retry", "err", err) return } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index 19ec70f2ed6..84294bde68c 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -427,7 +427,7 @@ func checkSwapControllerAvailability(ctx context.Context) bool { // memory.swap.max does not exist in the cgroup root, so we check /sys/fs/cgroup//memory.swap.max cm, err := libcontainercgroups.ParseCgroupFile("/proc/self/cgroup") if err != nil { - logger.V(5).Error(fmt.Errorf("failed to parse /proc/self/cgroup: %w", err), warn) + logger.V(5).Info(warn, "err", fmt.Errorf("failed to parse /proc/self/cgroup: %w", err)) return false } // For cgroup v2 unified hierarchy, there are no per-controller @@ -438,7 +438,7 @@ func checkSwapControllerAvailability(ctx context.Context) bool { } if _, err := os.Stat(p); err != nil { if !errors.Is(err, os.ErrNotExist) { - logger.V(5).Error(err, warn) + logger.V(5).Info(warn, "err", err) } return false } diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index e65c9a74d1c..9ff769a648e 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -82,7 +82,7 @@ func (hr *handlerRunner) Run(ctx context.Context, containerID kubecontainer.Cont output, err := hr.commandRunner.RunInContainer(ctx, containerID, handler.Exec.Command, 0) if err != nil { msg = fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - error: %v, message: %q", handler.Exec.Command, container.Name, format.Pod(pod), err, string(output)) - logger.V(1).Error(err, "Exec lifecycle hook for Container in Pod failed", "execCommand", handler.Exec.Command, "containerName", container.Name, "pod", klog.KObj(pod), "message", string(output)) + logger.V(1).Info("Exec lifecycle hook for Container in Pod failed", "execCommand", handler.Exec.Command, "containerName", container.Name, "pod", klog.KObj(pod), "message", string(output), "err", err) } return msg, err case handler.HTTPGet != nil: @@ -90,7 +90,7 @@ func (hr *handlerRunner) Run(ctx context.Context, containerID kubecontainer.Cont var msg string if err != nil { msg = fmt.Sprintf("HTTP lifecycle hook (%s) for Container %q in Pod %q failed - error: %v", handler.HTTPGet.Path, container.Name, format.Pod(pod), err) - logger.V(1).Error(err, "HTTP lifecycle hook for Container in Pod failed", "path", handler.HTTPGet.Path, "containerName", container.Name, "pod", klog.KObj(pod)) + logger.V(1).Info("HTTP lifecycle hook for Container in Pod failed", "path", handler.HTTPGet.Path, "containerName", container.Name, "pod", klog.KObj(pod), "err", err) } return msg, err case handler.Sleep != nil: @@ -98,7 +98,7 @@ func (hr *handlerRunner) Run(ctx context.Context, containerID kubecontainer.Cont var msg string if err != nil { msg = fmt.Sprintf("Sleep lifecycle hook (%d) for Container %q in Pod %q failed - error: %v", handler.Sleep.Seconds, container.Name, format.Pod(pod), err) - logger.V(1).Error(err, "Sleep lifecycle hook for Container in Pod failed", "sleepSeconds", handler.Sleep.Seconds, "containerName", container.Name, "pod", klog.KObj(pod)) + logger.V(1).Info("Sleep lifecycle hook for Container in Pod failed", "sleepSeconds", handler.Sleep.Seconds, "containerName", container.Name, "pod", klog.KObj(pod), "err", err) } return msg, err default: @@ -149,7 +149,7 @@ func (hr *handlerRunner) runHTTPHandler(ctx context.Context, pod *v1.Pod, contai discardHTTPRespBody(resp) if isHTTPResponseError(err) { - logger.V(1).Error(err, "HTTPS request to lifecycle hook got HTTP response, retrying with HTTP.", "pod", klog.KObj(pod), "host", req.URL.Host) + logger.V(1).Info("HTTPS request to lifecycle hook got HTTP response, retrying with HTTP.", "pod", klog.KObj(pod), "host", req.URL.Host, "err", err) req := req.Clone(ctx) req.URL.Scheme = "http" diff --git a/pkg/kubelet/metrics/collectors/cri_metrics.go b/pkg/kubelet/metrics/collectors/cri_metrics.go index 9ae50db409a..f8fd35e74d3 100644 --- a/pkg/kubelet/metrics/collectors/cri_metrics.go +++ b/pkg/kubelet/metrics/collectors/cri_metrics.go @@ -107,7 +107,7 @@ func (c *criMetricsCollector) criMetricToProm(logger klog.Logger, m *runtimeapi. desc, ok := c.descriptors[m.Name] if !ok { err := fmt.Errorf("error converting CRI Metric to prometheus format") - logger.V(5).Error(err, "Descriptor not present in pre-populated list of descriptors", "name", m.Name) + logger.V(5).Info("Descriptor not present in pre-populated list of descriptors", "name", m.Name, "err", err) return nil, err } diff --git a/pkg/kubelet/pleg/generic.go b/pkg/kubelet/pleg/generic.go index f89a6d02bd5..5db6ae40e4a 100644 --- a/pkg/kubelet/pleg/generic.go +++ b/pkg/kubelet/pleg/generic.go @@ -297,7 +297,7 @@ func (g *GenericPLEG) Relist() { status, updated, err := g.updateCache(ctx, pod, pid) if err != nil { // Rely on updateCache calling GetPodStatus to log the actual error. - g.logger.V(4).Error(err, "PLEG: Ignoring events for pod", "pod", klog.KRef(pod.Namespace, pod.Name)) + g.logger.V(4).Info("PLEG: Ignoring events for pod", "pod", klog.KRef(pod.Namespace, pod.Name), "err", err) // make sure we try to reinspect the pod during the next relisting needsReinspection[pid] = pod diff --git a/pkg/kubelet/prober/prober.go b/pkg/kubelet/prober/prober.go index 0441a11c8ec..f3ba5272c61 100644 --- a/pkg/kubelet/prober/prober.go +++ b/pkg/kubelet/prober/prober.go @@ -103,7 +103,7 @@ func (pb *prober) probe(ctx context.Context, probeType probeType, pod *v1.Pod, s if err != nil { // Handle probe error - logger.V(1).Error(err, "Probe errored", "probeType", probeType, "pod", klog.KObj(pod), "podUID", pod.UID, "containerName", container.Name, "probeResult", result) + logger.V(1).Info("Probe errored", "probeType", probeType, "pod", klog.KObj(pod), "podUID", pod.UID, "containerName", container.Name, "probeResult", result, "err", err) pb.recordContainerEvent(ctx, pod, &container, v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe errored and resulted in %s state: %s", probeType, result, err) return results.Failure, err } diff --git a/pkg/kubelet/prober/prober_manager.go b/pkg/kubelet/prober/prober_manager.go index 85737b6ad43..7595c0f7c57 100644 --- a/pkg/kubelet/prober/prober_manager.go +++ b/pkg/kubelet/prober/prober_manager.go @@ -194,7 +194,7 @@ func (m *manager) AddPod(ctx context.Context, pod *v1.Pod) { if c.StartupProbe != nil { key.probeType = startup if _, ok := m.workers[key]; ok { - logger.V(8).Error(nil, "Startup probe already exists for container", + logger.V(8).Info("Startup probe already exists for container", "pod", klog.KObj(pod), "containerName", c.Name) return } @@ -206,7 +206,7 @@ func (m *manager) AddPod(ctx context.Context, pod *v1.Pod) { if c.ReadinessProbe != nil { key.probeType = readiness if _, ok := m.workers[key]; ok { - logger.V(8).Error(nil, "Readiness probe already exists for container", + logger.V(8).Info("Readiness probe already exists for container", "pod", klog.KObj(pod), "containerName", c.Name) return } @@ -218,7 +218,7 @@ func (m *manager) AddPod(ctx context.Context, pod *v1.Pod) { if c.LivenessProbe != nil { key.probeType = liveness if _, ok := m.workers[key]; ok { - logger.V(8).Error(nil, "Liveness probe already exists for container", + logger.V(8).Info("Liveness probe already exists for container", "pod", klog.KObj(pod), "containerName", c.Name) return } diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index d4e7097df74..dfc6ef30d20 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -165,9 +165,7 @@ func (p *criStatsProvider) listPodStats(ctx context.Context, updateCPUNanoCoreUs return nil, err } // CRI implementation doesn't support ListPodSandboxStats, warn and fallback. - logger.V(5).Error(err, - "CRI implementation must be updated to support ListPodSandboxStats if PodAndContainerStatsFromCRI feature gate is enabled. Falling back to populating with cAdvisor; this call will fail in the future.", - ) + logger.V(5).Info("CRI implementation must be updated to support ListPodSandboxStats if PodAndContainerStatsFromCRI feature gate is enabled. Falling back to populating with cAdvisor; this call will fail in the future.", "err", err) } return p.listPodStatsPartiallyFromCRI(ctx, updateCPUNanoCoreUsage, containerMap, podSandboxMap, &rootFsInfo) } diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 8825ad7fef0..9631a7d9d57 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -499,7 +499,7 @@ func makePodStorageStats(logger klog.Logger, s *statsapi.PodStats, rootFsInfo *c logStats, err := hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) if err != nil { - logger.V(6).Error(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) + logger.V(6).Info("Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName), "err", err) // If people do in-place upgrade, there might be pods still using // the old log path. For those pods, no pod log stats is returned. // We should continue generating other stats in that case. @@ -507,7 +507,7 @@ func makePodStorageStats(logger klog.Logger, s *statsapi.PodStats, rootFsInfo *c } etcHostsStats, err := hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) if err != nil { - logger.V(6).Error(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) + logger.V(6).Info("Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName), "err", err) } s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, isCRIStatsProvider) } From e89362f1db15b4ca1fb6a3e6489e885448ccb0bc Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 6 Jan 2026 15:24:07 +0000 Subject: [PATCH 2/2] fix(kubelet): convert V().Error() to V().Info() in volumemanager Apply the same verbosity-aware logging fix to volumemanager components: - desired_state_of_world.go (2 instances) - reconciler/reconstruct.go (1 instance) This completes the kubelet-wide fix for V().Error() ignoring verbosity. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- pkg/kubelet/volumemanager/cache/desired_state_of_world.go | 4 ++-- pkg/kubelet/volumemanager/reconciler/reconstruct.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 0dfc390070e..e61e859fe85 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -670,7 +670,7 @@ func handleSELinuxMetricError(logger klog.Logger, err error, seLinuxSupported bo // This is not an error yet, but it will be when support for other access modes is added. warningMetric.Add(1.0) - logger.V(4).Error(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + logger.V(4).Info("Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file", "err", err) return nil } @@ -685,7 +685,7 @@ func getVolumePluginNameWithDriver(logger klog.Logger, plugin volume.VolumePlugi driverName, err := csi.GetCSIDriverName(spec) if err != nil { // In theory this is unreachable - such volume would not pass validation. - logger.V(4).Error(err, "failed to get CSI driver name from volume spec") + logger.V(4).Info("failed to get CSI driver name from volume spec", "err", err) driverName = "unknown" } // `/` is used to separate plugin + CSI driver in util.GetUniqueVolumeName() too diff --git a/pkg/kubelet/volumemanager/reconciler/reconstruct.go b/pkg/kubelet/volumemanager/reconciler/reconstruct.go index 8ac331328f1..30edb1c9fc3 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconstruct.go +++ b/pkg/kubelet/volumemanager/reconciler/reconstruct.go @@ -185,7 +185,7 @@ func (rc *reconciler) updateReconstructedFromNodeStatus(ctx context.Context) { node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(ctx, string(rc.nodeName), metav1.GetOptions{}) if fetchErr != nil { // This may repeat few times per second until kubelet is able to read its own status for the first time. - logger.V(4).Error(fetchErr, "Failed to get Node status to reconstruct device paths") + logger.V(4).Info("Failed to get Node status to reconstruct device paths", "err", fetchErr) return }