From 214a0ee7b8ca6d3528123e2bf9ddfbc2233f4b16 Mon Sep 17 00:00:00 2001 From: Aditi Sharma Date: Wed, 16 Mar 2022 18:01:26 +0530 Subject: [PATCH 1/2] Migrate container runtime endpoint flag to config Signed-off-by: Aditi Sharma Signed-off-by: Paco Xu --- cmd/kubelet/app/options/options.go | 11 ++++------- cmd/kubelet/app/server.go | 6 +++--- cmd/kubemark/hollow-node.go | 2 +- pkg/generated/openapi/zz_generated.openapi.go | 16 ++++++++++++++++ pkg/kubelet/apis/config/fuzzer/fuzzer.go | 2 ++ pkg/kubelet/apis/config/helpers_test.go | 2 ++ .../KubeletConfiguration/after/v1beta1.yaml | 1 + .../roundtrip/default/v1beta1.yaml | 2 ++ pkg/kubelet/apis/config/types.go | 11 +++++++++++ pkg/kubelet/apis/config/v1beta1/defaults.go | 3 +++ .../config/v1beta1/zz_generated.conversion.go | 4 ++++ pkg/kubelet/kubelet.go | 16 ++++------------ pkg/kubemark/hollow_kubelet.go | 2 +- .../src/k8s.io/kubelet/config/v1beta1/types.go | 10 ++++++++++ 14 files changed, 64 insertions(+), 24 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 238d6ae9f03..5f47756ea4f 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -98,10 +98,6 @@ type KubeletFlags struct { // Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities WindowsPriorityClass string - // remoteRuntimeEndpoint is the endpoint of remote runtime service - RemoteRuntimeEndpoint string - // remoteImageEndpoint is the endpoint of remote image service - RemoteImageEndpoint string // experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path ExperimentalMounterPath string // This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable. @@ -323,9 +319,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).") - fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "The endpoint of remote runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") - fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "The endpoint of remote image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") - // EXPERIMENTAL FLAGS bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels) fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf(" Labels to add when registering the node in the cluster. Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))) @@ -399,6 +392,10 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.") fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)") + // runtime flags + fs.StringVar(&c.ContainerRuntimeEndpoint, "container-runtime-endpoint", c.ContainerRuntimeEndpoint, "The endpoint of container runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") + fs.StringVar(&c.ImageServiceEndpoint, "image-service-endpoint", c.ImageServiceEndpoint, "The endpoint of container image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") + // Authentication fs.BoolVar(&c.Authentication.Anonymous.Enabled, "anonymous-auth", c.Authentication.Anonymous.Enabled, ""+ "Enables anonymous requests to the Kubelet server. Requests that are not rejected by another "+ diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 9cf57be9722..5b49c84b1e4 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -644,8 +644,8 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend } if kubeDeps.CAdvisorInterface == nil { - imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.RemoteRuntimeEndpoint) - kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.RemoteRuntimeEndpoint), s.LocalStorageCapacityIsolation) + imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntimeEndpoint) + kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntimeEndpoint), s.LocalStorageCapacityIsolation) if err != nil { return err } @@ -775,7 +775,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend klog.InfoS("Failed to ApplyOOMScoreAdj", "err", err) } - err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps, s.RemoteRuntimeEndpoint, s.RemoteImageEndpoint) + err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps) if err != nil { return err } diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index b60d3f40acb..02f11c99000 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -254,7 +254,7 @@ func run(cmd *cobra.Command, config *hollowNodeConfig) error { var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService if config.UseHostImageService { - imageService, err = remote.NewRemoteImageService(f.RemoteImageEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) + imageService, err = remote.NewRemoteImageService(c.ContainerRuntimeEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) if err != nil { return fmt.Errorf("Failed to init image service, error: %w", err) } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a1787cb9877..3a848e2a729 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -58629,7 +58629,23 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen Format: "", }, }, + "containerRuntimeEndpoint": { + SchemaProps: spec.SchemaProps{ + Description: "ContainerRuntimeEndpoint is the endpoint of container runtime. unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "imageServiceEndpoint": { + SchemaProps: spec.SchemaProps{ + Description: "ImageServiceEndpoint is the endpoint of container image service. If not specified the default value is ContainerRuntimeEndpoint", + Type: []string{"string"}, + Format: "", + }, + }, }, + Required: []string{"containerRuntimeEndpoint"}, }, }, Dependencies: []string{ diff --git a/pkg/kubelet/apis/config/fuzzer/fuzzer.go b/pkg/kubelet/apis/config/fuzzer/fuzzer.go index 67aea014442..f3d6de4324d 100644 --- a/pkg/kubelet/apis/config/fuzzer/fuzzer.go +++ b/pkg/kubelet/apis/config/fuzzer/fuzzer.go @@ -107,6 +107,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch" obj.AllowedUnsafeSysctls = []string{} obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir + obj.ContainerRuntimeEndpoint = "containerd.sock" + if obj.Logging.Format == "" { obj.Logging.Format = "text" } diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index f791a36dc87..d0e42167091 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -281,6 +281,8 @@ var ( "ShutdownGracePeriod.Duration", "ShutdownGracePeriodCriticalPods.Duration", "MemoryThrottlingFactor", + "ContainerRuntimeEndpoint", + "ImageServiceEndpoint", "Tracing.Endpoint", "Tracing.SamplingRatePerMillion", "LocalStorageCapacityIsolation", diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index cac43f8e1f7..f346d547bc5 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -17,6 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi +containerRuntimeEndpoint: "" contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index cac43f8e1f7..0b4f35b30a1 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -17,6 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi +containerRuntimeEndpoint: "" contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms @@ -42,6 +43,7 @@ httpCheckFrequency: 20s imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 imageMinimumGCAge: 2m0s +imageServiceEndpoint: containerd.sock iptablesDropBit: 15 iptablesMasqueradeBit: 14 kind: KubeletConfiguration diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 04432b9fc69..a9ceb7d29b4 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -450,6 +450,7 @@ type KubeletConfiguration struct { // registerNode enables automatic registration with the apiserver. // +optional RegisterNode bool + // Tracing specifies the versioned configuration for OpenTelemetry tracing clients. // See https://kep.k8s.io/2832 for more details. // +featureGate=KubeletTracing @@ -465,6 +466,16 @@ type KubeletConfiguration struct { // disabled. Once disabled, user should not set request/limit for container's ephemeral storage, or sizeLimit for emptyDir. // +optional LocalStorageCapacityIsolation bool + + // ContainerRuntimeEndpoint is the endpoint of container runtime. + // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + ContainerRuntimeEndpoint string + + // ImageServiceEndpoint is the endpoint of container image service. + // If not specified the default value is ContainerRuntimeEndpoint + // +optional + ImageServiceEndpoint string } // KubeletAuthorizationMode denotes the authorization mode for the kubelet diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 4b9397b734f..68eea1079b8 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -264,4 +264,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.LocalStorageCapacityIsolation == nil { obj.LocalStorageCapacityIsolation = utilpointer.BoolPtr(true) } + if obj.ImageServiceEndpoint == "" && obj.ContainerRuntimeEndpoint != "" { + obj.ImageServiceEndpoint = obj.ContainerRuntimeEndpoint + } } diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 3f289820445..43f08cc63c2 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -512,6 +512,8 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in if err := v1.Convert_Pointer_bool_To_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil { return err } + out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint + out.ImageServiceEndpoint = in.ImageServiceEndpoint return nil } @@ -691,6 +693,8 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in if err := v1.Convert_bool_To_Pointer_bool(&in.LocalStorageCapacityIsolation, &out.LocalStorageCapacityIsolation, s); err != nil { return err } + out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint + out.ImageServiceEndpoint = in.ImageServiceEndpoint return nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fac27f9a9df..052c75bf387 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -300,24 +300,16 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku } // PreInitRuntimeService will init runtime service before RunKubelet. -func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, - kubeDeps *Dependencies, - remoteRuntimeEndpoint string, - remoteImageEndpoint string) error { - // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified - if remoteRuntimeEndpoint != "" && remoteImageEndpoint == "" { - remoteImageEndpoint = remoteRuntimeEndpoint - } - +func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies) error { var err error - if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { + if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(kubeCfg.ContainerRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } - if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { + if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(kubeCfg.ImageServiceEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } - kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(remoteRuntimeEndpoint) + kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(kubeCfg.ContainerRuntimeEndpoint) return nil } diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 825b87e16a3..4adba49b503 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -158,7 +158,6 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, * f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels f.RegisterSchedulable = true - f.RemoteImageEndpoint = "unix:///run/containerd/containerd.sock" // Config struct c, err := options.NewKubeletConfiguration() @@ -166,6 +165,7 @@ func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, * panic(err) } + c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock" c.StaticPodURL = "" c.EnableServer = true c.Address = "0.0.0.0" /* bind address */ diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 3fab1abbe9c..d73dcddd655 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -803,6 +803,16 @@ type KubeletConfiguration struct { // Default: true // +optional LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"` + + // ContainerRuntimeEndpoint is the endpoint of container runtime. + // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"` + + // ImageServiceEndpoint is the endpoint of container image service. + // If not specified the default value is ContainerRuntimeEndpoint + // +optional + ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` } type KubeletAuthorizationMode string From f28f40e52134cc819f511eabb59357eb97a8a623 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Wed, 12 Oct 2022 02:09:19 +0800 Subject: [PATCH 2/2] remove a flag check that was introduced in #112542; address several comments Signed-off-by: Paco Xu --- cmd/kubelet/app/options/options.go | 7 ------- cmd/kubelet/app/options/options_test.go | 3 +-- cmd/kubemark/hollow-node.go | 2 +- pkg/generated/openapi/zz_generated.openapi.go | 4 ++-- pkg/kubelet/apis/config/fuzzer/fuzzer.go | 2 +- .../testdata/KubeletConfiguration/after/v1beta1.yaml | 2 +- .../KubeletConfiguration/roundtrip/default/v1beta1.yaml | 3 +-- pkg/kubelet/apis/config/types.go | 2 +- pkg/kubelet/apis/config/v1beta1/defaults.go | 4 ++-- pkg/kubelet/apis/config/v1beta1/defaults_test.go | 6 ++++++ pkg/kubelet/apis/config/validation/validation.go | 4 ++++ pkg/kubelet/apis/config/validation/validation_test.go | 1 + pkg/kubelet/kubelet.go | 6 +++++- staging/src/k8s.io/kubelet/config/v1beta1/types.go | 8 +++++--- 14 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 5f47756ea4f..8be25ccdc4a 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -189,13 +189,6 @@ func ValidateKubeletFlags(f *KubeletFlags) error { return fmt.Errorf("unsupported CRI runtime: %q, only %q is currently supported", f.ContainerRuntime, kubetypes.RemoteContainerRuntime) } - // Note: maybe we can test it for being a valid socket address as an additional improvement. - // The only problem with it will be that some setups may not specify 'unix://' prefix. - // So just check empty for back compat. - if f.RemoteRuntimeEndpoint == "" { - return fmt.Errorf("the container runtime endpoint address was not specified or empty, use --container-runtime-endpoint to set") - } - return nil } diff --git a/cmd/kubelet/app/options/options_test.go b/cmd/kubelet/app/options/options_test.go index 13d8e32bca2..e306d7ea914 100644 --- a/cmd/kubelet/app/options/options_test.go +++ b/cmd/kubelet/app/options/options_test.go @@ -183,8 +183,7 @@ func TestValidateKubeletFlags(t *testing.T) { ContainerRuntimeOptions: config.ContainerRuntimeOptions{ ContainerRuntime: kubetypes.RemoteContainerRuntime, }, - RemoteRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - NodeLabels: tt.labels, + NodeLabels: tt.labels, }) if tt.error && err == nil { diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 02f11c99000..773cdebd019 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -254,7 +254,7 @@ func run(cmd *cobra.Command, config *hollowNodeConfig) error { var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService if config.UseHostImageService { - imageService, err = remote.NewRemoteImageService(c.ContainerRuntimeEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) + imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) if err != nil { return fmt.Errorf("Failed to init image service, error: %w", err) } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 3a848e2a729..fefddbc2a82 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -58631,7 +58631,7 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen }, "containerRuntimeEndpoint": { SchemaProps: spec.SchemaProps{ - Description: "ContainerRuntimeEndpoint is the endpoint of container runtime. unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime", + Description: "ContainerRuntimeEndpoint is the endpoint of container runtime. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'", Default: "", Type: []string{"string"}, Format: "", @@ -58639,7 +58639,7 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen }, "imageServiceEndpoint": { SchemaProps: spec.SchemaProps{ - Description: "ImageServiceEndpoint is the endpoint of container image service. If not specified the default value is ContainerRuntimeEndpoint", + Description: "ImageServiceEndpoint is the endpoint of container image service. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'. If not specified, the value in containerRuntimeEndpoint is used.", Type: []string{"string"}, Format: "", }, diff --git a/pkg/kubelet/apis/config/fuzzer/fuzzer.go b/pkg/kubelet/apis/config/fuzzer/fuzzer.go index f3d6de4324d..b00dd10bcb8 100644 --- a/pkg/kubelet/apis/config/fuzzer/fuzzer.go +++ b/pkg/kubelet/apis/config/fuzzer/fuzzer.go @@ -107,7 +107,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch" obj.AllowedUnsafeSysctls = []string{} obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir - obj.ContainerRuntimeEndpoint = "containerd.sock" + obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock" if obj.Logging.Format == "" { obj.Logging.Format = "text" diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index f346d547bc5..401cc7b8eb4 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -17,7 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi -containerRuntimeEndpoint: "" +containerRuntimeEndpoint: unix:///run/containerd/containerd.sock contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index 0b4f35b30a1..401cc7b8eb4 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -17,7 +17,7 @@ cgroupsPerQOS: true configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi -containerRuntimeEndpoint: "" +containerRuntimeEndpoint: unix:///run/containerd/containerd.sock contentType: application/vnd.kubernetes.protobuf cpuCFSQuota: true cpuCFSQuotaPeriod: 100ms @@ -43,7 +43,6 @@ httpCheckFrequency: 20s imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 imageMinimumGCAge: 2m0s -imageServiceEndpoint: containerd.sock iptablesDropBit: 15 iptablesMasqueradeBit: 14 kind: KubeletConfiguration diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index a9ceb7d29b4..e0a30fe26db 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -469,7 +469,7 @@ type KubeletConfiguration struct { // ContainerRuntimeEndpoint is the endpoint of container runtime. // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. - // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime' ContainerRuntimeEndpoint string // ImageServiceEndpoint is the endpoint of container image service. diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 68eea1079b8..018368d4e51 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -264,7 +264,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.LocalStorageCapacityIsolation == nil { obj.LocalStorageCapacityIsolation = utilpointer.BoolPtr(true) } - if obj.ImageServiceEndpoint == "" && obj.ContainerRuntimeEndpoint != "" { - obj.ImageServiceEndpoint = obj.ContainerRuntimeEndpoint + if obj.ContainerRuntimeEndpoint == "" { + obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock" } } diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index 447e5f5dad3..6c01d476730 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -78,6 +78,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: utilpointer.Int32Ptr(85), ImageGCLowThresholdPercent: utilpointer.Int32Ptr(80), + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, CgroupsPerQOS: utilpointer.BoolPtr(true), CgroupDriver: "cgroupfs", @@ -173,6 +174,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeStatusUpdateFrequency: zeroDuration, NodeStatusReportFrequency: zeroDuration, NodeLeaseDurationSeconds: 0, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", ImageMinimumGCAge: zeroDuration, ImageGCHighThresholdPercent: utilpointer.Int32(0), ImageGCLowThresholdPercent: utilpointer.Int32(0), @@ -285,6 +287,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second}, NodeStatusReportFrequency: metav1.Duration{Duration: 5 * time.Minute}, NodeLeaseDurationSeconds: 40, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: utilpointer.Int32(0), ImageGCLowThresholdPercent: utilpointer.Int32(0), @@ -394,6 +397,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, ImageGCHighThresholdPercent: utilpointer.Int32(1), ImageGCLowThresholdPercent: utilpointer.Int32(1), @@ -538,6 +542,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, ImageGCHighThresholdPercent: utilpointer.Int32(1), ImageGCLowThresholdPercent: utilpointer.Int32(1), @@ -674,6 +679,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeStatusUpdateFrequency: metav1.Duration{Duration: 1 * time.Minute}, NodeStatusReportFrequency: metav1.Duration{Duration: 1 * time.Minute}, NodeLeaseDurationSeconds: 40, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: utilpointer.Int32Ptr(85), ImageGCLowThresholdPercent: utilpointer.Int32Ptr(80), diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index dc1c2b7c007..aa4c2c5fcfa 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -257,5 +257,9 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur allErrors = append(allErrors, fmt.Errorf("invalid configuration: memoryThrottlingFactor %v must be greater than 0 and less than or equal to 1.0", *kc.MemoryThrottlingFactor)) } + if kc.ContainerRuntimeEndpoint == "" { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: the containerRuntimeEndpoint was not specified or empty")) + } + return utilerrors.NewAggregate(allErrors) } diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index 81de56d554b..71cd2bdddab 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -73,6 +73,7 @@ var ( Logging: logsapi.LoggingConfiguration{ Format: "text", }, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", } ) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 052c75bf387..8a8156f507d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -301,11 +301,15 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku // PreInitRuntimeService will init runtime service before RunKubelet. func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies) error { + remoteImageEndpoint := kubeCfg.ImageServiceEndpoint + if remoteImageEndpoint == "" && kubeCfg.ContainerRuntimeEndpoint != "" { + remoteImageEndpoint = kubeCfg.ContainerRuntimeEndpoint + } var err error if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(kubeCfg.ContainerRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } - if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(kubeCfg.ImageServiceEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { + if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration, kubeDeps.TracerProvider); err != nil { return err } diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index d73dcddd655..79bca2b3459 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -805,12 +805,14 @@ type KubeletConfiguration struct { LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"` // ContainerRuntimeEndpoint is the endpoint of container runtime. - // unix domain sockets supported on Linux while npipes and tcp endpoints are supported for windows. - // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime + // Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime' ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"` // ImageServiceEndpoint is the endpoint of container image service. - // If not specified the default value is ContainerRuntimeEndpoint + // Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. + // Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'. + // If not specified, the value in containerRuntimeEndpoint is used. // +optional ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` }