diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 238d6ae9f03..5f47756ea4f 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -98,10 +98,6 @@ type KubeletFlags struct { // Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities WindowsPriorityClass string - // remoteRuntimeEndpoint is the endpoint of remote runtime service - RemoteRuntimeEndpoint string - // remoteImageEndpoint is the endpoint of remote image service - RemoteImageEndpoint string // experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path ExperimentalMounterPath string // This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable. @@ -323,9 +319,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).") - fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "The endpoint of remote runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") - fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "The endpoint of remote image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") - // EXPERIMENTAL FLAGS bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels) fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf(" Labels to add when registering the node in the cluster. Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))) @@ -399,6 +392,10 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.") fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)") + // runtime flags + fs.StringVar(&c.ContainerRuntimeEndpoint, "container-runtime-endpoint", c.ContainerRuntimeEndpoint, "The endpoint of container runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") + fs.StringVar(&c.ImageServiceEndpoint, "image-service-endpoint", c.ImageServiceEndpoint, "The endpoint of container image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") + // Authentication fs.BoolVar(&c.Authentication.Anonymous.Enabled, "anonymous-auth", c.Authentication.Anonymous.Enabled, ""+ "Enables anonymous requests to the Kubelet server. Requests that are not rejected by another "+ diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 9cf57be9722..5b49c84b1e4 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -644,8 +644,8 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend } if kubeDeps.CAdvisorInterface == nil { - imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.RemoteRuntimeEndpoint) - kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.RemoteRuntimeEndpoint), s.LocalStorageCapacityIsolation) + imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntimeEndpoint) + kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntimeEndpoint), s.LocalStorageCapacityIsolation) if err != nil { return err } @@ -775,7 +775,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend klog.InfoS("Failed to ApplyOOMScoreAdj", "err", err) } - err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps, s.RemoteRuntimeEndpoint, s.RemoteImageEndpoint) + err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps) if err != nil { return err } diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index b60d3f40acb..02f11c99000 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -254,7 +254,7 @@ func run(cmd *cobra.Command, config *hollowNodeConfig) error { var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService if config.UseHostImageService { - imageService, err = remote.NewRemoteImageService(f.RemoteImageEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) + imageService, err = remote.NewRemoteImageService(c.ContainerRuntimeEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) if err != nil { return fmt.Errorf("Failed to init image service, error: %w", err) } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a1787cb9877..3a848e2a729 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -58629,7 +58629,23 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen Format: "", }, }, + "containerRuntimeEndpoint": { + SchemaProps: spec.SchemaProps{ + Description: "ContainerRuntimeEndpoint is the endpoint of container runtime. unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "imageServiceEndpoint": { + SchemaProps: spec.SchemaProps{ + Description: "ImageServiceEndpoint is the endpoint of container image service. If not specified the default value is ContainerRuntimeEndpoint", + Type: []string{"string"}, + Format: "", + }, + }, }, + Required: []string{"containerRuntimeEndpoint"}, }, }, Dependencies: []string{ diff --git a/pkg/kubelet/apis/config/fuzzer/fuzzer.go b/pkg/kubelet/apis/config/fuzzer/fuzzer.go index 67aea014442..f3d6de4324d 100644 --- a/pkg/kubelet/apis/config/fuzzer/fuzzer.go +++ b/pkg/kubelet/apis/config/fuzzer/fuzzer.go @@ -107,6 +107,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch" obj.AllowedUnsafeSysctls = []string{} obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir + obj.ContainerRuntimeEndpoint = "containerd.sock" + if obj.Logging.Format == "" { obj.Logging.Format = "text" } diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index f791a36dc87..d0e42167091 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -281,6 +281,8 @@ var ( "ShutdownGracePeriod.Duration", "ShutdownGracePeriodCriticalPods.Duration", "MemoryThrottlingFactor", + "ContainerRuntimeEndpoint", + "ImageServiceEndpoint", "Tracing.Endpoint", "Tracing.SamplingRatePerMillion", "LocalStorageCapacityIsolation", diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index cac43f8e1f7..f346d547bc5 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -17,6 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi +containerRuntimeEndpoint: "" contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index cac43f8e1f7..0b4f35b30a1 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -17,6 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi +containerRuntimeEndpoint: "" contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms @@ -42,6 +43,7 @@ httpCheckFrequency: 20s imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 imageMinimumGCAge: 2m0s +imageServiceEndpoint: containerd.sock iptablesDropBit: 15 iptablesMasqueradeBit: 14 kind: KubeletConfiguration diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 04432b9fc69..a9ceb7d29b4 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -450,6 +450,7 @@ type KubeletConfiguration struct { // registerNode enables automatic registration with the apiserver. // +optional RegisterNode bool + // Tracing specifies the versioned configuration for OpenTelemetry tracing clients. // See https://kep.k8s.io/2832 for more details. // +featureGate=KubeletTracing @@ -465,6 +466,16 @@ type KubeletConfiguration struct { // disabled. Once disabled, user should not set request/limit for container's ephemeral storage, or sizeLimit for emptyDir. // +optional LocalStorageCapacityIsolation bool + + // ContainerRuntimeEndpoint is the endpoint of container runtime. + // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + ContainerRuntimeEndpoint string + + // ImageServiceEndpoint is the endpoint of container image service. + // If not specified the default value is ContainerRuntimeEndpoint + // +optional + ImageServiceEndpoint string } // KubeletAuthorizationMode denotes the authorization mode for the kubelet diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 4b9397b734f..68eea1079b8 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -264,4 +264,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.LocalStorageCapacityIsolation == nil { obj.LocalStorageCapacityIsolation = utilpointer.BoolPtr(true) } + if obj.ImageServiceEndpoint == "" && obj.ContainerRuntimeEndpoint != "" { + obj.ImageServiceEndpoint = obj.ContainerRuntimeEndpoint + } } diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 3f289820445..43f08cc63c2 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -512,6 +512,8 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in if err := v1.Convert_Pointer_bool_To_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil { return err } + out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint + out.ImageServiceEndpoint = in.ImageServiceEndpoint return nil } @@ -691,6 +693,8 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in if err := v1.Convert_bool_To_Pointer_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil { return err } + out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint + out.ImageServiceEndpoint = in.ImageServiceEndpoint return nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fac27f9a9df..052c75bf387 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -300,24 +300,16 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku } // PreInitRuntimeService will init runtime service before RunKubelet. -func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, - kubeDeps *Dependencies, - remoteRuntimeEndpoint string, - remoteImageEndpoint string) error { - // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified - if remoteRuntimeEndpoint != "" && remoteImageEndpoint == "" { - remoteImageEndpoint = remoteRuntimeEndpoint - } - +func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies) error { var err error - if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { + if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(kubeCfg.ContainerRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } - if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { + if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(kubeCfg.ImageServiceEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } - kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(remoteRuntimeEndpoint) + kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(kubeCfg.ContainerRuntimeEndpoint) return nil } diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 825b87e16a3..4adba49b503 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -158,7 +158,6 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, * f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels f.RegisterSchedulable = true - f.RemoteImageEndpoint = "unix:///run/containerd/containerd.sock" // Config struct c, err := options.NewKubeletConfiguration() @@ -166,6 +165,7 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, * panic(err) } + c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock" c.StaticPodURL = "" c.EnableServer = true c.Address = "0.0.0.0" /* bind address */ diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 3fab1abbe9c..d73dcddd655 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -803,6 +803,16 @@ type KubeletConfiguration struct { // Default: true // +optional LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"` + + // ContainerRuntimeEndpoint is the endpoint of container runtime. + // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"` + + // ImageServiceEndpoint is the endpoint of container image service. + // If not specified the default value is ContainerRuntimeEndpoint + // +optional + ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` } type KubeletAuthorizationMode string