Merge pull request #112136 from pacoxu/migrate-runtime-endpoint-flags

kubelet: migrate container runtime endpoint flag to config

cmd/kubelet/app/options/options.go

@@ -98,10 +98,6 @@ type KubeletFlags struct {
 	// Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities
 	WindowsPriorityClass string
 
-	// remoteRuntimeEndpoint is the endpoint of remote runtime service
-	RemoteRuntimeEndpoint string
-	// remoteImageEndpoint is the endpoint of remote image service
-	RemoteImageEndpoint string
 	// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
 	ExperimentalMounterPath string
 	// This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable.
@@ -193,13 +189,6 @@ func ValidateKubeletFlags(f *KubeletFlags) error {
 		return fmt.Errorf("unsupported CRI runtime: %q, only %q is currently supported", f.ContainerRuntime, kubetypes.RemoteContainerRuntime)
 	}
 
-	// Note: maybe we can test it for being a valid socket address as an additional improvement.
-	// The only problem with it will be that some setups may not specify 'unix://' prefix.
-	// So just check empty for back compat.
-	if f.RemoteRuntimeEndpoint == "" {
-		return fmt.Errorf("the container runtime endpoint address was not specified or empty, use --container-runtime-endpoint to set")
-	}
-
 	return nil
 }
 
@@ -323,9 +312,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
 
 	fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
 
-	fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "The endpoint of remote runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
-	fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "The endpoint of remote image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
-
 	// EXPERIMENTAL FLAGS
 	bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels)
 	fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf("<Warning: Alpha feature> Labels to add when registering the node in the cluster.  Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")))
@@ -399,6 +385,10 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig
 	fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.")
 	fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)")
 
+	// runtime flags
+	fs.StringVar(&c.ContainerRuntimeEndpoint, "container-runtime-endpoint", c.ContainerRuntimeEndpoint, "The endpoint of container runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
+	fs.StringVar(&c.ImageServiceEndpoint, "image-service-endpoint", c.ImageServiceEndpoint, "The endpoint of container image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
+
 	// Authentication
 	fs.BoolVar(&c.Authentication.Anonymous.Enabled, "anonymous-auth", c.Authentication.Anonymous.Enabled, ""+
 		"Enables anonymous requests to the Kubelet server. Requests that are not rejected by another "+

cmd/kubelet/app/options/options_test.go

@@ -183,8 +183,7 @@ func TestValidateKubeletFlags(t *testing.T) {
 				ContainerRuntimeOptions: config.ContainerRuntimeOptions{
 					ContainerRuntime: kubetypes.RemoteContainerRuntime,
 				},
-				RemoteRuntimeEndpoint: "unix:///run/containerd/containerd.sock",
+				NodeLabels: tt.labels,
-				NodeLabels:            tt.labels,
 			})
 
 			if tt.error && err == nil {

cmd/kubelet/app/server.go

@@ -644,8 +644,8 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 	}
 
 	if kubeDeps.CAdvisorInterface == nil {
-		imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.RemoteRuntimeEndpoint)
+		imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntimeEndpoint)
-		kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.RemoteRuntimeEndpoint), s.LocalStorageCapacityIsolation)
+		kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntimeEndpoint), s.LocalStorageCapacityIsolation)
 		if err != nil {
 			return err
 		}
@@ -775,7 +775,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 		klog.InfoS("Failed to ApplyOOMScoreAdj", "err", err)
 	}
 
-	err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps, s.RemoteRuntimeEndpoint, s.RemoteImageEndpoint)
+	err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps)
 	if err != nil {
 		return err
 	}

cmd/kubemark/hollow-node.go

@@ -254,7 +254,7 @@ func run(cmd *cobra.Command, config *hollowNodeConfig) error {
 
 		var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
 		if config.UseHostImageService {
-			imageService, err = remote.NewRemoteImageService(f.RemoteImageEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
+			imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
 			if err != nil {
 				return fmt.Errorf("Failed to init image service, error: %w", err)
 			}

pkg/generated/openapi/zz_generated.openapi.go

@@ -58632,7 +58632,23 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen
 							Format:      "",
 						},
 					},
+					"containerRuntimeEndpoint": {
+						SchemaProps: spec.SchemaProps{
+							Description: "ContainerRuntimeEndpoint is the endpoint of container runtime. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'",
+							Default:     "",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"imageServiceEndpoint": {
+						SchemaProps: spec.SchemaProps{
+							Description: "ImageServiceEndpoint is the endpoint of container image service. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'. If not specified, the value in containerRuntimeEndpoint is used.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
 				},
+				Required: []string{"containerRuntimeEndpoint"},
 			},
 		},
 		Dependencies: []string{

pkg/kubelet/apis/config/fuzzer/fuzzer.go

@@ -107,6 +107,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch"
 			obj.AllowedUnsafeSysctls = []string{}
 			obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir
+			obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"
+
 			if obj.Logging.Format == "" {
 				obj.Logging.Format = "text"
 			}

pkg/kubelet/apis/config/helpers_test.go

@@ -281,6 +281,8 @@ var (
 		"ShutdownGracePeriod.Duration",
 		"ShutdownGracePeriodCriticalPods.Duration",
 		"MemoryThrottlingFactor",
+		"ContainerRuntimeEndpoint",
+		"ImageServiceEndpoint",
 		"Tracing.Endpoint",
 		"Tracing.SamplingRatePerMillion",
 		"LocalStorageCapacityIsolation",

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml

@@ -17,6 +17,7 @@ cgroupsPerQOS: true
 configMapAndSecretChangeDetectionStrategy: Watch
 containerLogMaxFiles: 5
 containerLogMaxSize: 10Mi
+containerRuntimeEndpoint: unix:///run/containerd/containerd.sock
 contentType: application/vnd.kubernetes.protobuf
 cpuCFSQuota: true
 cpuCFSQuotaPeriod: 100ms

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml

@@ -17,6 +17,7 @@ cgroupsPerQOS: true
 configMapAndSecretChangeDetectionStrategy: Watch
 containerLogMaxFiles: 5
 containerLogMaxSize: 10Mi
+containerRuntimeEndpoint: unix:///run/containerd/containerd.sock
 contentType: application/vnd.kubernetes.protobuf
 cpuCFSQuota: true
 cpuCFSQuotaPeriod: 100ms

pkg/kubelet/apis/config/types.go

@@ -450,6 +450,7 @@ type KubeletConfiguration struct {
 	// registerNode enables automatic registration with the apiserver.
 	// +optional
 	RegisterNode bool
+
 	// Tracing specifies the versioned configuration for OpenTelemetry tracing clients.
 	// See https://kep.k8s.io/2832 for more details.
 	// +featureGate=KubeletTracing
@@ -465,6 +466,16 @@ type KubeletConfiguration struct {
 	// disabled. Once disabled, user should not set request/limit for container's ephemeral storage, or sizeLimit for emptyDir.
 	// +optional
 	LocalStorageCapacityIsolation bool
+
+	// ContainerRuntimeEndpoint is the endpoint of container runtime.
+	// unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows.
+	// Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'
+	ContainerRuntimeEndpoint string
+
+	// ImageServiceEndpoint is the endpoint of container image service.
+	// If not specified the default value is ContainerRuntimeEndpoint
+	// +optional
+	ImageServiceEndpoint string
 }
 
 // KubeletAuthorizationMode denotes the authorization mode for the kubelet

pkg/kubelet/apis/config/v1beta1/defaults.go

@@ -264,4 +264,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
 	if obj.LocalStorageCapacityIsolation == nil {
 		obj.LocalStorageCapacityIsolation = utilpointer.BoolPtr(true)
 	}
+	if obj.ContainerRuntimeEndpoint == "" {
+		obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"
+	}
 }

pkg/kubelet/apis/config/v1beta1/defaults_test.go

@@ -78,6 +78,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ImageMinimumGCAge:                         metav1.Duration{Duration: 2 * time.Minute},
 				ImageGCHighThresholdPercent:               utilpointer.Int32Ptr(85),
 				ImageGCLowThresholdPercent:                utilpointer.Int32Ptr(80),
+				ContainerRuntimeEndpoint:                  "unix:///run/containerd/containerd.sock",
 				VolumeStatsAggPeriod:                      metav1.Duration{Duration: time.Minute},
 				CgroupsPerQOS:                             utilpointer.BoolPtr(true),
 				CgroupDriver:                              "cgroupfs",
@@ -173,6 +174,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeStatusUpdateFrequency:        zeroDuration,
 				NodeStatusReportFrequency:        zeroDuration,
 				NodeLeaseDurationSeconds:         0,
+				ContainerRuntimeEndpoint:         "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:                zeroDuration,
 				ImageGCHighThresholdPercent:      utilpointer.Int32(0),
 				ImageGCLowThresholdPercent:       utilpointer.Int32(0),
@@ -285,6 +287,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeStatusUpdateFrequency:        metav1.Duration{Duration: 10 * time.Second},
 				NodeStatusReportFrequency:        metav1.Duration{Duration: 5 * time.Minute},
 				NodeLeaseDurationSeconds:         40,
+				ContainerRuntimeEndpoint:         "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:                metav1.Duration{Duration: 2 * time.Minute},
 				ImageGCHighThresholdPercent:      utilpointer.Int32(0),
 				ImageGCLowThresholdPercent:       utilpointer.Int32(0),
@@ -394,6 +397,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeStatusUpdateFrequency:      metav1.Duration{Duration: 60 * time.Second},
 				NodeStatusReportFrequency:      metav1.Duration{Duration: 60 * time.Second},
 				NodeLeaseDurationSeconds:       1,
+				ContainerRuntimeEndpoint:       "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:              metav1.Duration{Duration: 60 * time.Second},
 				ImageGCHighThresholdPercent:    utilpointer.Int32(1),
 				ImageGCLowThresholdPercent:     utilpointer.Int32(1),
@@ -538,6 +542,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeStatusUpdateFrequency:      metav1.Duration{Duration: 60 * time.Second},
 				NodeStatusReportFrequency:      metav1.Duration{Duration: 60 * time.Second},
 				NodeLeaseDurationSeconds:       1,
+				ContainerRuntimeEndpoint:       "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:              metav1.Duration{Duration: 60 * time.Second},
 				ImageGCHighThresholdPercent:    utilpointer.Int32(1),
 				ImageGCLowThresholdPercent:     utilpointer.Int32(1),
@@ -674,6 +679,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeStatusUpdateFrequency:                 metav1.Duration{Duration: 1 * time.Minute},
 				NodeStatusReportFrequency:                 metav1.Duration{Duration: 1 * time.Minute},
 				NodeLeaseDurationSeconds:                  40,
+				ContainerRuntimeEndpoint:                  "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:                         metav1.Duration{Duration: 2 * time.Minute},
 				ImageGCHighThresholdPercent:               utilpointer.Int32Ptr(85),
 				ImageGCLowThresholdPercent:                utilpointer.Int32Ptr(80),

pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go

@@ -512,6 +512,8 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
 	if err := v1.Convert_Pointer_bool_To_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil {
 		return err
 	}
+	out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint
+	out.ImageServiceEndpoint = in.ImageServiceEndpoint
 	return nil
 }
 
@@ -691,6 +693,8 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
 	if err := v1.Convert_bool_To_Pointer_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil {
 		return err
 	}
+	out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint
+	out.ImageServiceEndpoint = in.ImageServiceEndpoint
 	return nil
 }
 

pkg/kubelet/apis/config/validation/validation.go

@@ -257,5 +257,9 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
 		allErrors = append(allErrors, fmt.Errorf("invalid configuration: memoryThrottlingFactor %v must be greater than 0 and less than or equal to 1.0", *kc.MemoryThrottlingFactor))
 	}
 
+	if kc.ContainerRuntimeEndpoint == "" {
+		allErrors = append(allErrors, fmt.Errorf("invalid configuration: the containerRuntimeEndpoint was not specified or empty"))
+	}
+
 	return utilerrors.NewAggregate(allErrors)
 }

pkg/kubelet/apis/config/validation/validation_test.go

@@ -73,6 +73,7 @@ var (
 		Logging: logsapi.LoggingConfiguration{
 			Format: "text",
 		},
+		ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock",
 	}
 )
 

pkg/kubelet/kubelet.go

@@ -302,24 +302,20 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku
 }
 
 // PreInitRuntimeService will init runtime service before RunKubelet.
-func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
+func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies) error {
-	kubeDeps *Dependencies,
+	remoteImageEndpoint := kubeCfg.ImageServiceEndpoint
-	remoteRuntimeEndpoint string,
+	if remoteImageEndpoint == "" && kubeCfg.ContainerRuntimeEndpoint != "" {
-	remoteImageEndpoint string) error {
+		remoteImageEndpoint = kubeCfg.ContainerRuntimeEndpoint
-	// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
-	if remoteRuntimeEndpoint != "" && remoteImageEndpoint == "" {
-		remoteImageEndpoint = remoteRuntimeEndpoint
 	}
-
 	var err error
-	if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil {
+	if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(kubeCfg.ContainerRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil {
 		return err
 	}
 	if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil {
 		return err
 	}
 
-	kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(remoteRuntimeEndpoint)
+	kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(kubeCfg.ContainerRuntimeEndpoint)
 
 	return nil
 }

pkg/kubemark/hollow_kubelet.go

@@ -158,7 +158,6 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *
 	f.MaxPerPodContainerCount = 2
 	f.NodeLabels = opt.NodeLabels
 	f.RegisterSchedulable = true
-	f.RemoteImageEndpoint = "unix:///run/containerd/containerd.sock"
 
 	// Config struct
 	c, err := options.NewKubeletConfiguration()
@@ -166,6 +165,7 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *
 		panic(err)
 	}
 
+	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
 	c.StaticPodURL = ""
 	c.EnableServer = true
 	c.Address = "0.0.0.0" /* bind address */

staging/src/k8s.io/kubelet/config/v1beta1/types.go

@@ -803,6 +803,18 @@ type KubeletConfiguration struct {
 	// Default: true
 	// +optional
 	LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"`
+
+	// ContainerRuntimeEndpoint is the endpoint of container runtime.
+	// Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows.
+	// Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'
+	ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"`
+
+	// ImageServiceEndpoint is the endpoint of container image service.
+	// Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows.
+	// Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'.
+	// If not specified, the value in containerRuntimeEndpoint is used.
+	// +optional
+	ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"`
 }
 
 type KubeletAuthorizationMode string