Merge pull request #137326 from amritansh1502/fix-136333-kubelet-zero-psi-metrics

Fix zero PSI metrics emitted when OS doesn't enable PSI

cmd/kubelet/app/server.go

@@ -782,7 +782,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 
 	if kubeDeps.CAdvisorInterface == nil {
 		imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntimeEndpoint)
-		kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntimeEndpoint), s.LocalStorageCapacityIsolation)
+		kubeDeps.CAdvisorInterface, err = cadvisor.New(klog.FromContext(ctx), imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntimeEndpoint), s.LocalStorageCapacityIsolation)
 		if err != nil {
 			return err
 		}

pkg/kubelet/cadvisor/cadvisor_linux.go

@@ -46,6 +46,8 @@ import (
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	"github.com/google/cadvisor/manager"
 	"github.com/google/cadvisor/utils/sysfs"
+	"github.com/opencontainers/cgroups"
+	cgroupfs2 "github.com/opencontainers/cgroups/fs2"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/features"
@@ -88,7 +90,7 @@ func init() {
 }
 
 // New creates a new cAdvisor Interface for linux systems.
-func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
+func New(logger klog.Logger, imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
 	sysFs := sysfs.NewRealSysFs()
 
 	includedMetrics := cadvisormetrics.MetricSet{
@@ -102,7 +104,11 @@ func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots [
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
-		includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
+		if IsPsiEnabled(logger) {
+			includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
+		} else {
+			logger.Info("PSI support not available")
+		}
 	}
 
 	if usingLegacyStats || localStorageCapacityIsolation {
@@ -162,6 +168,26 @@ func (cc *cadvisorClient) ImagesFsInfo(ctx context.Context) (cadvisorapiv2.FsInf
 	return cc.getFsInfo(ctx, label)
 }
 
+// IsPsiEnabled checks whether PSI (Pressure Stall Information) is available on
+// the host by opening the root cgroup's cpu.pressure file using the same
+// opencontainers/cgroups library that cAdvisor uses to read actual PSI values.
+// PSI is a single kernel feature (CONFIG_PSI / boot param "psi=") so checking
+// cpu.pressure alone is sufficient to determine support for all three resources
+// (cpu, memory, io).
+func IsPsiEnabled(logger klog.Logger) bool {
+	return isPsiEnabled(logger, cgroupfs2.UnifiedMountpoint, "cpu.pressure")
+}
+
+func isPsiEnabled(logger klog.Logger, cgroupDir, psiFile string) bool {
+	f, err := cgroups.OpenFile(cgroupDir, psiFile, os.O_RDONLY)
+	if err != nil {
+		logger.V(4).Info("PSI not available", "dir", cgroupDir, "file", psiFile, "err", err)
+		return false
+	}
+	_ = f.Close()
+	return true
+}
+
 func (cc *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
 	return cc.GetDirFsInfo(cc.rootPath)
 }

pkg/kubelet/cadvisor/cadvisor_linux_test.go

@@ -20,14 +20,52 @@ package cadvisor
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/google/cadvisor/container/crio"
 	cadvisorfs "github.com/google/cadvisor/fs"
+	"github.com/opencontainers/cgroups"
+	"k8s.io/klog/v2"
 )
 
+func TestIsPsiEnabled(t *testing.T) {
+	testcases := []struct {
+		description   string
+		createPSIFile bool
+		expected      bool
+	}{{
+		description:   "PSI enabled when cgroup pressure file exists",
+		createPSIFile: true,
+		expected:      true,
+	}, {
+		description:   "PSI disabled when cgroup pressure file does not exist",
+		createPSIFile: false,
+		expected:      false,
+	}}
+
+	cgroups.TestMode = true
+	defer func() { cgroups.TestMode = false }()
+
+	for _, tc := range testcases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+
+			if tc.createPSIFile {
+				err := os.WriteFile(filepath.Join(tmpDir, "cpu.pressure"), []byte("some avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"), 0644)
+				require.NoError(t, err)
+			}
+
+			result := isPsiEnabled(klog.Background(), tmpDir, "cpu.pressure")
+			assert.Equal(t, tc.expected, result)
+		})
+	}
+}
+
 func TestImageFsInfoLabel(t *testing.T) {
 	testcases := []struct {
 		description     string

pkg/kubelet/cadvisor/cadvisor_unsupported.go

@@ -24,6 +24,7 @@ import (
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	"k8s.io/klog/v2"
 )
 
 type cadvisorUnsupported struct {
@@ -32,7 +33,7 @@ type cadvisorUnsupported struct {
 var _ Interface = new(cadvisorUnsupported)
 
 // New creates a new cAdvisor Interface for unsupported systems.
-func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupsRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
+func New(_ klog.Logger, imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupsRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
 	return &cadvisorUnsupported{}, nil
 }
 
@@ -73,3 +74,7 @@ func (cu *cadvisorUnsupported) ContainerFsInfo(context.Context) (cadvisorapiv2.F
 func (cu *cadvisorUnsupported) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
 	return cadvisorapiv2.FsInfo{}, nil
 }
+
+func IsPsiEnabled(_ klog.Logger) bool {
+	return false
+}

pkg/kubelet/cadvisor/cadvisor_windows.go

@@ -23,7 +23,6 @@ import (
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
-
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/kubelet/winstats"
 )
@@ -36,7 +35,7 @@ type cadvisorClient struct {
 var _ Interface = new(cadvisorClient)
 
 // New creates a cAdvisor and exports its API on the specified port if port > 0.
-func New(imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
+func New(_ klog.Logger, imageFsInfoProvider ImageFsInfoProvider, rootPath string, cgroupRoots []string, usingLegacyStats, localStorageCapacityIsolation bool) (Interface, error) {
 	client, err := winstats.NewPerfCounterClient(klog.TODO())
 	return &cadvisorClient{
 		rootPath:       rootPath,
@@ -80,3 +79,7 @@ func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
 func (cu *cadvisorClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
 	return cu.winStatsClient.GetDirFsInfo(path)
 }
+
+func IsPsiEnabled(_ klog.Logger) bool {
+	return false
+}

pkg/kubelet/server/server.go

@@ -90,6 +90,7 @@ import (
 	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	apisgrpc "k8s.io/kubernetes/pkg/kubelet/apis/grpc"
 	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
+	kubeletcadvisor "k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
 	"k8s.io/kubernetes/pkg/kubelet/prober"
@@ -484,7 +485,9 @@ func (s *Server) InstallAuthNotRequiredHandlers(ctx context.Context) {
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPSI) {
-		includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
+		if kubeletcadvisor.IsPsiEnabled(logger) {
+			includedMetrics[cadvisormetrics.PressureMetrics] = struct{}{}
+		}
 	}
 
 	// cAdvisor metrics are exposed under the secured handler as well