From 83a0d0c660a490e0d9b1d0de0409948f92b5cb88 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Wed, 23 Jul 2025 11:10:09 -0400 Subject: [PATCH 1/2] kubelet: add metric for version CRI implementation will lose support Signed-off-by: Peter Hunt --- cmd/kubelet/app/server.go | 2 + pkg/kubelet/metrics/metrics.go | 11 +++ test/e2e_node/cgroup_driver_from_cri_test.go | 83 ++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 test/e2e_node/cgroup_driver_from_cri_test.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index c39be3c0fe1..81c46b7fb26 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1397,6 +1397,8 @@ func getCgroupDriverFromCRI(ctx context.Context, s *options.KubeletServer, kubeD continue } // CRI implementation doesn't support RuntimeConfig, fallback + legacyregistry.MustRegister(kubeletmetrics.CRILosingSupport) + kubeletmetrics.CRILosingSupport.WithLabelValues("1.36.0").Inc() logger.Info("CRI implementation should be updated to support RuntimeConfig when KubeletCgroupDriverFromCRI feature gate has been enabled. Falling back to using cgroupDriver from kubelet config.") return nil } diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 684c912f420..05d496681d9 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -74,6 +74,7 @@ const ( RestartedPodTotalKey = "restarted_pods_total" ImagePullDurationKey = "image_pull_duration_seconds" CgroupVersionKey = "cgroup_version" + CRILosingSupportKey = "cri_losing_support" // Metrics keys of remote runtime operations RuntimeOperationsKey = "runtime_operations_total" @@ -1018,6 +1019,16 @@ var ( }, ) + CRILosingSupport = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: CRILosingSupportKey, + Help: "the Kubernetes version that the currently running CRI implementation will lose support on if not upgraded.", + StabilityLevel: metrics.ALPHA, + }, + []string{"version"}, + ) + // DRAOperationsDuration tracks the duration of the DRA PrepareResources and UnprepareResources requests. DRAOperationsDuration = metrics.NewHistogramVec( &metrics.HistogramOpts{ diff --git a/test/e2e_node/cgroup_driver_from_cri_test.go b/test/e2e_node/cgroup_driver_from_cri_test.go new file mode 100644 index 00000000000..d40143ca988 --- /dev/null +++ b/test/e2e_node/cgroup_driver_from_cri_test.go @@ -0,0 +1,83 @@ +//go:build linux +// +build linux + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" + "k8s.io/kubernetes/test/e2e/feature" + "k8s.io/kubernetes/test/e2e/framework" + e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" + "k8s.io/kubernetes/test/e2e_node/criproxy" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = SIGDescribe("Cgroup Driver From CRI", feature.CriProxy, framework.WithSerial(), func() { + f := framework.NewDefaultFramework("cgroup-driver-from-cri") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + ginkgo.Context("", func() { + ginkgo.BeforeEach(func() { + if err := resetCRIProxyInjector(e2eCriProxy); err != nil { + ginkgo.Skip("Skip the test since the CRI Proxy is undefined.") + } + ginkgo.DeferCleanup(func() error { + return resetCRIProxyInjector(e2eCriProxy) + }) + }) + + ginkgo.It("should only report a metric if CRI is outdated", func(ctx context.Context) { + expectedErr := status.Error(codes.Unimplemented, "unimplemented") + err := addCRIProxyInjector(e2eCriProxy, func(apiName string) error { + if apiName == criproxy.RuntimeConfig { + return expectedErr + } + return nil + }) + framework.ExpectNoError(err) + restartKubelet(context.Background(), true) + time.Sleep(time.Second * 1) + + m, err := e2emetrics.GrabKubeletMetricsWithoutProxy(context.Background(), nodeNameOrIP()+":10255", "/metrics") + framework.ExpectNoError(err) + samples := m[kubeletmetrics.KubeletSubsystem+"_"+kubeletmetrics.CRILosingSupportKey] + + gomega.Expect(samples).NotTo(gomega.BeEmpty()) + gomega.Expect(samples[0].Metric["version"]).To(gomega.BeEquivalentTo("1.36.0")) + }) + ginkgo.It("should not emit metric if CRI is new enough", func() { + restartKubelet(context.Background(), true) + time.Sleep(time.Second * 1) + + m, err := e2emetrics.GrabKubeletMetricsWithoutProxy(context.Background(), nodeNameOrIP()+":10255", "/metrics") + framework.ExpectNoError(err) + samples := m[kubeletmetrics.KubeletSubsystem+"_"+kubeletmetrics.CRILosingSupportKey] + + gomega.Expect(samples).To(gomega.BeEmpty()) + }) + }) +}) From 5dc2030c027370f37077754af03f163debd9a999 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 22 Jul 2025 14:36:40 -0400 Subject: [PATCH 2/2] 4033: GA KubeletCgroupDriverFromCRI feature gate Signed-off-by: Peter Hunt --- cmd/kubelet/app/server.go | 8 +++----- pkg/features/kube_features.go | 1 + .../reference/versioned_feature_list.yaml | 4 ++++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 81c46b7fb26..f069f6b08b9 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -741,10 +741,8 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend } // Get cgroup driver setting from CRI - if utilfeature.DefaultFeatureGate.Enabled(features.KubeletCgroupDriverFromCRI) { - if err := getCgroupDriverFromCRI(ctx, s, kubeDeps); err != nil { - return err - } + if err := getCgroupDriverFromCRI(ctx, s, kubeDeps); err != nil { + return err } var cgroupRoots []string @@ -1399,7 +1397,7 @@ func getCgroupDriverFromCRI(ctx context.Context, s *options.KubeletServer, kubeD // CRI implementation doesn't support RuntimeConfig, fallback legacyregistry.MustRegister(kubeletmetrics.CRILosingSupport) kubeletmetrics.CRILosingSupport.WithLabelValues("1.36.0").Inc() - logger.Info("CRI implementation should be updated to support RuntimeConfig when KubeletCgroupDriverFromCRI feature gate has been enabled. Falling back to using cgroupDriver from kubelet config.") + logger.Info("CRI implementation should be updated to support RuntimeConfig. Falling back to using cgroupDriver from kubelet config.") return nil } } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 788eee524ca..0ec6df2fb7d 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1301,6 +1301,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate KubeletCgroupDriverFromCRI: { {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.37 }, KubeletCrashLoopBackOffMax: { diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 0ffd3beee2a..011a1594672 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -713,6 +713,10 @@ lockToDefault: false preRelease: Beta version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.34" - name: KubeletCrashLoopBackOffMax versionedSpecs: - default: false