From 541ff7cd1699c3a238ca7f24250eea66dad2a1bc Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Arango Gutierrez Date: Thu, 28 Aug 2025 14:46:49 +0200 Subject: [PATCH 1/2] DRA: Add configurable health check timeout per device Implements device-specific health check timeouts in the DRA health monitoring system as defined in KEP-4680. This allows DRA drivers to specify custom timeout values for individual devices through the gRPC health API. Changes: - Add HealthCheckTimeout field to state.DeviceHealth struct to store device-specific timeout durations - Add health_check_timeout_seconds field to DeviceHealth proto message in the DRA health gRPC API (v1alpha1) - Update manager.go to extract timeout from gRPC responses and apply DefaultHealthTimeout (30s) when not specified - Handle negative timeout values defensively by logging a warning and falling back to the default timeout - Simplify healthinfo.go by removing redundant fallback logic since timeouts are now always set at creation time - Update tests to include HealthCheckTimeout in test fixtures The timeout behavior is: - Positive values: Use the specified timeout in seconds - Zero or unspecified: Use DefaultHealthTimeout (30 seconds) - Negative values: Log warning and use DefaultHealthTimeout This implementation provides flexibility for DRA drivers to define appropriate health check intervals for different device types while maintaining backward compatibility through sensible defaults. Ref: KEP-4680 (Add Resource Health to Pod Status) Ref: kubernetes/enhancements#5476 Signed-off-by: Carlos Eduardo Arango Gutierrez --- pkg/kubelet/cm/dra/healthinfo.go | 22 +++++-- pkg/kubelet/cm/dra/healthinfo_test.go | 39 ++++++------ pkg/kubelet/cm/dra/manager.go | 26 ++++++-- pkg/kubelet/cm/dra/state/state.go | 5 ++ .../pkg/apis/dra-health/v1alpha1/api.pb.go | 63 ++++++++++++------- .../pkg/apis/dra-health/v1alpha1/api.proto | 5 ++ 6 files changed, 109 insertions(+), 51 deletions(-) diff --git a/pkg/kubelet/cm/dra/healthinfo.go b/pkg/kubelet/cm/dra/healthinfo.go index b389192c3b0..48982201bcf 100644 --- a/pkg/kubelet/cm/dra/healthinfo.go +++ b/pkg/kubelet/cm/dra/healthinfo.go @@ -28,9 +28,9 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" ) -// TODO(#133118): Make health timeout configurable. const ( - healthTimeout = 30 * time.Second + // DefaultHealthTimeout is the default timeout for device health checks when not specified by the plugin + DefaultHealthTimeout = 30 * time.Second ) // healthInfoCache is a cache of known device health. @@ -129,7 +129,14 @@ func (cache *healthInfoCache) getHealthInfo(driverName, poolName, deviceName str if driver, ok := (*cache.HealthInfo)[driverName]; ok { key := poolName + "/" + deviceName if device, ok := driver.Devices[key]; ok { - if now.Sub(device.LastUpdated) > healthTimeout { + // Use device-specific timeout if set, otherwise use default + timeout := device.HealthCheckTimeout + if timeout == 0 { + timeout = DefaultHealthTimeout + } + + // Check if device health has timed out + if now.Sub(device.LastUpdated) > timeout { res = state.DeviceHealthStatusUnknown } else { res = device.Health @@ -178,7 +185,14 @@ func (cache *healthInfoCache) updateHealthInfo(driverName string, devices []stat // them. Mark them as "Unknown" if their status has timed out. for key, existingDevice := range currentDriver.Devices { if _, wasReported := reportedKeys[key]; !wasReported { - if existingDevice.Health != state.DeviceHealthStatusUnknown && now.Sub(existingDevice.LastUpdated) > healthTimeout { + // Use device-specific timeout if set, otherwise use default + timeout := existingDevice.HealthCheckTimeout + if timeout == 0 { + timeout = DefaultHealthTimeout + } + + // Mark as unknown if the device health has timed out + if existingDevice.Health != state.DeviceHealthStatusUnknown && now.Sub(existingDevice.LastUpdated) > timeout { existingDevice.Health = state.DeviceHealthStatusUnknown existingDevice.LastUpdated = now currentDriver.Devices[key] = existingDevice diff --git a/pkg/kubelet/cm/dra/healthinfo_test.go b/pkg/kubelet/cm/dra/healthinfo_test.go index 3100b642d1b..947ef515f68 100644 --- a/pkg/kubelet/cm/dra/healthinfo_test.go +++ b/pkg/kubelet/cm/dra/healthinfo_test.go @@ -38,9 +38,10 @@ const ( var ( testDeviceHealth = state.DeviceHealth{ - PoolName: testPool, - DeviceName: testDevice, - Health: state.DeviceHealthStatusHealthy, + PoolName: testPool, + DeviceName: testDevice, + Health: state.DeviceHealthStatusHealthy, + HealthCheckTimeout: DefaultHealthTimeout, } ) @@ -206,7 +207,7 @@ func TestGetHealthInfo(t *testing.T) { driverState := (*cache.HealthInfo)[testDriver] deviceKey := testPool + "/" + testDevice device := driverState.Devices[deviceKey] - device.LastUpdated = time.Now().Add((-healthTimeout) - time.Second) + device.LastUpdated = time.Now().Add((-DefaultHealthTimeout) - time.Second) driverState.Devices[deviceKey] = device (*cache.HealthInfo)[testDriver] = driverState return nil @@ -237,7 +238,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device exists and is healthy", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -249,7 +250,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device exists and is unhealthy", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -261,7 +262,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device exists but timed out", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) - time.Second)}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) - time.Second), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -273,7 +274,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device exists, just within timeout", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) + time.Second)}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) + time.Second), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -285,7 +286,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device does not exist, just outside of timeout", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) - time.Second)}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) - time.Second), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -297,7 +298,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "device does not exist", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -309,7 +310,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "driver does not exist", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: "driver2", @@ -321,7 +322,7 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "pool does not exist", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -333,8 +334,8 @@ func TestGetHealthInfoRobust(t *testing.T) { name: "multiple devices", initialState: &state.DevicesHealthMap{ testDriver: {Devices: map[string]state.DeviceHealth{ - testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()}, - testPool + "/device2": {PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now()}, + testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, + testPool + "/device2": {PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout}, }}, }, driverName: testDriver, @@ -382,7 +383,7 @@ func TestUpdateHealthInfo(t *testing.T) { assert.Equal(t, state.DeviceHealthStatusUnhealthy, cache.getHealthInfo(testDriver, testPool, testDevice)) // 4 -- Add second device, omit first - secondDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusHealthy} + secondDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusHealthy, HealthCheckTimeout: DefaultHealthTimeout} // When the first device is omitted, it should be marked as "Unknown" after a timeout. // For this test, we simulate the timeout by not reporting it. firstDeviceAsUnknown := newHealth @@ -392,7 +393,7 @@ func TestUpdateHealthInfo(t *testing.T) { err = cache.withLock(func() error { deviceKey := testPool + "/" + testDevice device := (*cache.HealthInfo)[testDriver].Devices[deviceKey] - device.LastUpdated = time.Now().Add(-healthTimeout * 2) + device.LastUpdated = time.Now().Add(-DefaultHealthTimeout * 2) (*cache.HealthInfo)[testDriver].Devices[deviceKey] = device return nil }) @@ -411,7 +412,7 @@ func TestUpdateHealthInfo(t *testing.T) { assert.Equal(t, state.DeviceHealthStatusUnknown, cache2.getHealthInfo(testDriver, testPool, testDevice)) // 6 -- Test how updateHealthInfo handles device timeouts - timeoutDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: "Unhealthy"} + timeoutDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: "Unhealthy", HealthCheckTimeout: DefaultHealthTimeout} _, err = cache.updateHealthInfo(testDriver, []state.DeviceHealth{timeoutDevice}) require.NoError(t, err) @@ -420,14 +421,14 @@ func TestUpdateHealthInfo(t *testing.T) { driverState := (*cache.HealthInfo)[testDriver] deviceKey := testPool + "/timeoutDevice" device := driverState.Devices[deviceKey] - device.LastUpdated = time.Now().Add((-healthTimeout) - time.Second) + device.LastUpdated = time.Now().Add((-DefaultHealthTimeout) - time.Second) driverState.Devices[deviceKey] = device (*cache.HealthInfo)[testDriver] = driverState return nil }) require.NoError(t, err) - expectedTimeoutDeviceUnknown := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: state.DeviceHealthStatusUnknown} + expectedTimeoutDeviceUnknown := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: state.DeviceHealthStatusUnknown, HealthCheckTimeout: DefaultHealthTimeout} expectedChanged6 := []state.DeviceHealth{expectedTimeoutDeviceUnknown} changedDevices, err = cache.updateHealthInfo(testDriver, []state.DeviceHealth{}) require.NoError(t, err) diff --git a/pkg/kubelet/cm/dra/manager.go b/pkg/kubelet/cm/dra/manager.go index 9d034a091a5..892fe481f00 100644 --- a/pkg/kubelet/cm/dra/manager.go +++ b/pkg/kubelet/cm/dra/manager.go @@ -933,11 +933,29 @@ func (m *Manager) HandleWatchResourcesStream(ctx context.Context, stream draheal default: health = state.DeviceHealthStatusUnknown } + + // Extract the health check timeout from the gRPC response + // If not specified, zero, or negative, use the default timeout + timeout := DefaultHealthTimeout + timeoutSeconds := d.GetHealthCheckTimeoutSeconds() + if timeoutSeconds > 0 { + timeout = time.Duration(timeoutSeconds) * time.Second + } else if timeoutSeconds < 0 { + // Log warning for negative timeout values and use default + logger.V(4).Info("Ignoring negative health check timeout, using default", + "pluginName", pluginName, + "poolName", d.GetDevice().GetPoolName(), + "deviceName", d.GetDevice().GetDeviceName(), + "providedTimeout", timeoutSeconds, + "defaultTimeout", DefaultHealthTimeout) + } + devices[i] = state.DeviceHealth{ - PoolName: d.GetDevice().GetPoolName(), - DeviceName: d.GetDevice().GetDeviceName(), - Health: health, - LastUpdated: time.Unix(d.GetLastUpdatedTime(), 0), + PoolName: d.GetDevice().GetPoolName(), + DeviceName: d.GetDevice().GetDeviceName(), + Health: health, + LastUpdated: time.Unix(d.GetLastUpdatedTime(), 0), + HealthCheckTimeout: timeout, } } diff --git a/pkg/kubelet/cm/dra/state/state.go b/pkg/kubelet/cm/dra/state/state.go index 3f15d3c38e9..1b078879ef4 100644 --- a/pkg/kubelet/cm/dra/state/state.go +++ b/pkg/kubelet/cm/dra/state/state.go @@ -95,4 +95,9 @@ type DeviceHealth struct { // LastUpdated keeps track of the last health status update of this device. LastUpdated time.Time + + // HealthCheckTimeout is the timeout for the health check of the device. + // Zero value means use the default timeout (DefaultHealthTimeout). + // This ensures backward compatibility with existing data. + HealthCheckTimeout time.Duration } diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.pb.go index dc78d8a0ade..e2221a4c593 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.pb.go @@ -204,8 +204,12 @@ type DeviceHealth struct { Health HealthStatus `protobuf:"varint,2,opt,name=health,proto3,enum=v1alpha1.HealthStatus" json:"health,omitempty"` // The Unix time (in seconds) of when this health status was last determined by the plugin. LastUpdatedTime int64 `protobuf:"varint,3,opt,name=last_updated_time,json=lastUpdatedTime,proto3" json:"last_updated_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Health check timeout in seconds for this device. + // If not specified, zero, or negative, Kubelet will use a default timeout. + // Negative values will be logged and treated as if not specified. + HealthCheckTimeoutSeconds int64 `protobuf:"varint,4,opt,name=health_check_timeout_seconds,json=healthCheckTimeoutSeconds,proto3" json:"health_check_timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *DeviceHealth) Reset() { @@ -259,6 +263,13 @@ func (x *DeviceHealth) GetLastUpdatedTime() int64 { return 0 } +func (x *DeviceHealth) GetHealthCheckTimeoutSeconds() int64 { + if x != nil { + return x.HealthCheckTimeoutSeconds + } + return 0 +} + // NodeWatchResourcesResponse contains a list of devices and their current health. // This should be a complete list for the driver; Kubelet will reconcile this // state with its internal cache. Any devices managed by the driver that are @@ -321,7 +332,7 @@ var file_staging_src_k8s_io_kubelet_pkg_apis_dra_health_v1alpha1_api_proto_rawDe 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x9e, 0x01, 0x0a, + 0x52, 0x0a, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xdf, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x32, 0x0a, 0x06, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x49, @@ -331,27 +342,31 @@ var file_staging_src_k8s_io_kubelet_pkg_apis_dra_health_v1alpha1_api_proto_rawDe 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6c, 0x61, - 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x4e, 0x0a, - 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x07, 0x64, - 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x52, 0x07, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2a, 0x37, 0x0a, - 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, - 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x48, 0x45, - 0x41, 0x4c, 0x54, 0x48, 0x59, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x48, 0x45, 0x41, - 0x4c, 0x54, 0x48, 0x59, 0x10, 0x02, 0x32, 0x78, 0x0a, 0x11, 0x44, 0x52, 0x41, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x63, 0x0a, 0x12, 0x4e, - 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x12, 0x23, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4e, 0x6f, 0x64, - 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, - 0x42, 0x2d, 0x5a, 0x2b, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x6c, - 0x65, 0x74, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x64, 0x72, 0x61, 0x2d, - 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3f, 0x0a, + 0x1c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x19, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x4e, + 0x0a, 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x07, + 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x07, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2a, 0x37, + 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, + 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x48, + 0x45, 0x41, 0x4c, 0x54, 0x48, 0x59, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x48, 0x45, + 0x41, 0x4c, 0x54, 0x48, 0x59, 0x10, 0x02, 0x32, 0x78, 0x0a, 0x11, 0x44, 0x52, 0x41, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x63, 0x0a, 0x12, + 0x4e, 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4e, 0x6f, + 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, + 0x01, 0x42, 0x2d, 0x5a, 0x2b, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x6b, 0x75, 0x62, 0x65, + 0x6c, 0x65, 0x74, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x64, 0x72, 0x61, + 0x2d, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.proto index 08d83d89ea1..1a25781704b 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra-health/v1alpha1/api.proto @@ -56,6 +56,11 @@ message DeviceHealth { // The Unix time (in seconds) of when this health status was last determined by the plugin. int64 last_updated_time = 3; + + // Health check timeout in seconds for this device. + // If not specified, zero, or negative, Kubelet will use a default timeout. + // Negative values will be logged and treated as if not specified. + int64 health_check_timeout_seconds = 4; } // NodeWatchResourcesResponse contains a list of devices and their current health. From 374baacf92c0949fc8e50fb9b16823c72530d24d Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Wed, 5 Nov 2025 11:32:18 -0500 Subject: [PATCH 2/2] Check HealthCheckTimeout in updateHealthInfo comparison Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- pkg/kubelet/cm/dra/healthinfo.go | 6 +++--- pkg/kubelet/cm/dra/manager_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kubelet/cm/dra/healthinfo.go b/pkg/kubelet/cm/dra/healthinfo.go index 48982201bcf..47f7ef12c7b 100644 --- a/pkg/kubelet/cm/dra/healthinfo.go +++ b/pkg/kubelet/cm/dra/healthinfo.go @@ -134,7 +134,7 @@ func (cache *healthInfoCache) getHealthInfo(driverName, poolName, deviceName str if timeout == 0 { timeout = DefaultHealthTimeout } - + // Check if device health has timed out if now.Sub(device.LastUpdated) > timeout { res = state.DeviceHealthStatusUnknown @@ -173,7 +173,7 @@ func (cache *healthInfoCache) updateHealthInfo(driverName string, devices []stat existingDevice, ok := currentDriver.Devices[key] - if !ok || existingDevice.Health != reportedDevice.Health { + if !ok || existingDevice.Health != reportedDevice.Health || existingDevice.HealthCheckTimeout != reportedDevice.HealthCheckTimeout { changedDevices = append(changedDevices, reportedDevice) } @@ -190,7 +190,7 @@ func (cache *healthInfoCache) updateHealthInfo(driverName string, devices []stat if timeout == 0 { timeout = DefaultHealthTimeout } - + // Mark as unknown if the device health has timed out if existingDevice.Health != state.DeviceHealthStatusUnknown && now.Sub(existingDevice.LastUpdated) > timeout { existingDevice.Health = state.DeviceHealthStatusUnknown diff --git a/pkg/kubelet/cm/dra/manager_test.go b/pkg/kubelet/cm/dra/manager_test.go index d9df5655839..67b11f3344f 100644 --- a/pkg/kubelet/cm/dra/manager_test.go +++ b/pkg/kubelet/cm/dra/manager_test.go @@ -1424,7 +1424,7 @@ func TestHandleWatchResourcesStream(t *testing.T) { manager, runStreamTest := setupNewManagerAndRunStreamTest(t, stCtx, initialClaim) // Pre-populate health cache - initialHealth := state.DeviceHealth{PoolName: poolName, DeviceName: deviceName, Health: "Unhealthy", LastUpdated: time.Now().Add(-5 * time.Millisecond)} // Ensure LastUpdated is slightly in past + initialHealth := state.DeviceHealth{PoolName: poolName, DeviceName: deviceName, Health: "Unhealthy", LastUpdated: time.Now().Add(-5 * time.Millisecond), HealthCheckTimeout: DefaultHealthTimeout} // Ensure LastUpdated is slightly in past _, err := manager.healthInfoCache.updateHealthInfo(driverName, []state.DeviceHealth{initialHealth}) require.NoError(t, err, "Failed to pre-populate health cache")