diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index 3364c14fd6e..c10a4c6f79b 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -96,6 +96,7 @@ KubeletConfiguration: clusterDNS: - 10.96.0.10 clusterDomain: cluster.local + configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1.yaml index 1ad3415af1d..196e4eca886 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1.yaml @@ -85,6 +85,7 @@ kubeletConfiguration: clusterDNS: - 10.96.0.10 clusterDomain: cluster.local + configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1_without_TypeMeta.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1_without_TypeMeta.yaml index dfaccd79534..c68a01aaf78 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1_without_TypeMeta.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha1_without_TypeMeta.yaml @@ -82,6 +82,7 @@ kubeletConfiguration: clusterDNS: - 10.96.0.10 clusterDomain: cluster.local + configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml index f8af262b3fb..b0c29b43553 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml @@ -86,6 +86,7 @@ kubeletConfiguration: clusterDNS: - 10.96.0.10 clusterDomain: cluster.local + configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml index 70ea5986e11..8f4465dc5f4 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml @@ -81,6 +81,7 @@ kubeletConfiguration: clusterDNS: - 10.192.0.10 clusterDomain: cluster.global + configMapAndSecretChangeDetectionStrategy: Watch containerLogMaxFiles: 5 containerLogMaxSize: 10Mi contentType: application/vnd.kubernetes.protobuf diff --git a/pkg/kubelet/apis/kubeletconfig/helpers_test.go b/pkg/kubelet/apis/kubeletconfig/helpers_test.go index 929ac004223..138765fe285 100644 --- a/pkg/kubelet/apis/kubeletconfig/helpers_test.go +++ b/pkg/kubelet/apis/kubeletconfig/helpers_test.go @@ -153,6 +153,7 @@ var ( "CgroupsPerQOS", "ClusterDNS[*]", "ClusterDomain", + "ConfigMapAndSecretChangeDetectionStrategy", "ContainerLogMaxFiles", "ContainerLogMaxSize", "ContentType", diff --git a/pkg/kubelet/apis/kubeletconfig/types.go b/pkg/kubelet/apis/kubeletconfig/types.go index 4b220981f36..1e4bb6d8aff 100644 --- a/pkg/kubelet/apis/kubeletconfig/types.go +++ b/pkg/kubelet/apis/kubeletconfig/types.go @@ -39,6 +39,23 @@ const ( HairpinNone = "none" ) +// ResourceChangeDetectionStrategy denotes a mode in which internal +// managers (secret, configmap) are discovering object changes. +type ResourceChangeDetectionStrategy string + +// Enum settings for different strategies of kubelet managers. +const ( + // GetChangeDetectionStrategy is a mode in which kubelet fetches + // necessary objects directly from apiserver. + GetChangeDetectionStrategy ResourceChangeDetectionStrategy = "Get" + // TTLCacheChangeDetectionStrategy is a mode in which kubelet uses + // ttl cache for object directly fetched from apiserver. + TTLCacheChangeDetectionStrategy ResourceChangeDetectionStrategy = "Cache" + // WatchChangeDetectionStrategy is a mode in which kubelet uses + // watches to observe changes to objects that are in its interest. + WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeletConfiguration contains the configuration for the Kubelet @@ -259,6 +276,8 @@ type KubeletConfiguration struct { ContainerLogMaxSize string // Maximum number of container log files that can be present for a container. ContainerLogMaxFiles int32 + // ConfigMapAndSecretChangeDetectionStrategy is a mode in which config map and secret managers are running. + ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy /* the following fields are meant for Node Allocatable */ diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/defaults.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/defaults.go index b8d6bb32f73..43a1302ae2a 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/defaults.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/defaults.go @@ -198,6 +198,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.ContainerLogMaxFiles == nil { obj.ContainerLogMaxFiles = utilpointer.Int32Ptr(5) } + if obj.ConfigMapAndSecretChangeDetectionStrategy == "" { + obj.ConfigMapAndSecretChangeDetectionStrategy = WatchChangeDetectionStrategy + } if obj.EnforceNodeAllocatable == nil { obj.EnforceNodeAllocatable = DefaultNodeAllocatableEnforcement } diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go index 2ec949efe4f..27b551ddfac 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go @@ -39,6 +39,23 @@ const ( HairpinNone = "none" ) +// ResourceChangeDetectionStrategy denotes a mode in which internal +// managers (secret, configmap) are discovering object changes. +type ResourceChangeDetectionStrategy string + +// Enum settings for different strategies of kubelet managers. +const ( + // GetChangeDetectionStrategy is a mode in which kubelet fetches + // necessary objects directly from apiserver. + GetChangeDetectionStrategy ResourceChangeDetectionStrategy = "Get" + // TTLCacheChangeDetectionStrategy is a mode in which kubelet uses + // ttl cache for object directly fetched from apiserver. + TTLCacheChangeDetectionStrategy ResourceChangeDetectionStrategy = "Cache" + // WatchChangeDetectionStrategy is a mode in which kubelet uses + // watches to observe changes to objects that are in its interest. + WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeletConfiguration contains the configuration for the Kubelet @@ -612,6 +629,11 @@ type KubeletConfiguration struct { // Default: 5 // +optional ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // ConfigMapAndSecretChangeDetectionStrategy is a mode in which + // config map and secret managers are running. + // Default: "Watching" + // +optional + ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy `json:"configMapAndSecretChangeDetectionStrategy,omitempty"` /* the following fields are meant for Node Allocatable */ diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go index 5765982b744..b392be32236 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go @@ -253,6 +253,7 @@ func autoConvert_v1beta1_KubeletConfiguration_To_kubeletconfig_KubeletConfigurat if err := v1.Convert_Pointer_int32_To_int32(&in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles, s); err != nil { return err } + out.ConfigMapAndSecretChangeDetectionStrategy = kubeletconfig.ResourceChangeDetectionStrategy(in.ConfigMapAndSecretChangeDetectionStrategy) out.SystemReserved = *(*map[string]string)(unsafe.Pointer(&in.SystemReserved)) out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved)) out.SystemReservedCgroup = in.SystemReservedCgroup @@ -377,6 +378,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1beta1_KubeletConfigurat if err := v1.Convert_int32_To_Pointer_int32(&in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles, s); err != nil { return err } + out.ConfigMapAndSecretChangeDetectionStrategy = ResourceChangeDetectionStrategy(in.ConfigMapAndSecretChangeDetectionStrategy) out.SystemReserved = *(*map[string]string)(unsafe.Pointer(&in.SystemReserved)) out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved)) out.SystemReservedCgroup = in.SystemReservedCgroup diff --git a/pkg/kubelet/configmap/BUILD b/pkg/kubelet/configmap/BUILD index 84d1e029d7f..ba777637f5b 100644 --- a/pkg/kubelet/configmap/BUILD +++ b/pkg/kubelet/configmap/BUILD @@ -15,12 +15,14 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/configmap", deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/apis/core/v1:go_default_library", "//pkg/kubelet/util/manager:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/pkg/kubelet/configmap/configmap_manager.go b/pkg/kubelet/configmap/configmap_manager.go index 91c75ddca72..76aa1f03fb8 100644 --- a/pkg/kubelet/configmap/configmap_manager.go +++ b/pkg/kubelet/configmap/configmap_manager.go @@ -23,12 +23,14 @@ import ( "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + corev1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/kubernetes/pkg/kubelet/util/manager" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/watch" ) type Manager interface { @@ -123,3 +125,25 @@ func NewCachingConfigMapManager(kubeClient clientset.Interface, getTTL manager.G manager: manager.NewCacheBasedManager(configMapStore, getConfigMapNames), } } + +// NewWatchingConfigMapManager creates a manager that keeps a cache of all configmaps +// necessary for registered pods. +// It implements the following logic: +// - whenever a pod is created or updated, we start inidvidual watches for all +// referenced objects that aren't referenced from other registered pods +// - every GetObject() returns a value from local cache propagated via watches +func NewWatchingConfigMapManager(kubeClient clientset.Interface) Manager { + listConfigMap := func(namespace string, opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.CoreV1().ConfigMaps(namespace).List(opts) + } + watchConfigMap := func(namespace string, opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.CoreV1().ConfigMaps(namespace).Watch(opts) + } + newConfigMap := func() runtime.Object { + return &v1.ConfigMap{} + } + gr := corev1.Resource("configmap") + return &configMapManager{ + manager: manager.NewWatchBasedManager(listConfigMap, watchConfigMap, newConfigMap, gr, getConfigMapNames), + } +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6777337bd4e..2c8307f16d8 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -551,12 +551,25 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.cloudproviderRequestTimeout = 10 * time.Second } - secretManager := secret.NewCachingSecretManager( - kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode)) - klet.secretManager = secretManager + var secretManager secret.Manager + var configMapManager configmap.Manager + switch kubeCfg.ConfigMapAndSecretChangeDetectionStrategy { + case kubeletconfiginternal.WatchChangeDetectionStrategy: + secretManager = secret.NewWatchingSecretManager(kubeDeps.KubeClient) + configMapManager = configmap.NewWatchingConfigMapManager(kubeDeps.KubeClient) + case kubeletconfiginternal.TTLCacheChangeDetectionStrategy: + secretManager = secret.NewCachingSecretManager( + kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode)) + configMapManager = configmap.NewCachingConfigMapManager( + kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode)) + case kubeletconfiginternal.GetChangeDetectionStrategy: + secretManager = secret.NewSimpleSecretManager(kubeDeps.KubeClient) + configMapManager = configmap.NewSimpleConfigMapManager(kubeDeps.KubeClient) + default: + return nil, fmt.Errorf("unknown configmap and secret manager mode: %v", kubeCfg.ConfigMapAndSecretChangeDetectionStrategy) + } - configMapManager := configmap.NewCachingConfigMapManager( - kubeDeps.KubeClient, manager.GetObjectTTLFromNodeFunc(klet.GetNode)) + klet.secretManager = secretManager klet.configMapManager = configMapManager if klet.experimentalHostUserNamespaceDefaulting { diff --git a/pkg/kubelet/secret/BUILD b/pkg/kubelet/secret/BUILD index 4b39334d0bb..9aeb025ef82 100644 --- a/pkg/kubelet/secret/BUILD +++ b/pkg/kubelet/secret/BUILD @@ -30,12 +30,14 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/secret", deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/apis/core/v1:go_default_library", "//pkg/kubelet/util/manager:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/pkg/kubelet/secret/secret_manager.go b/pkg/kubelet/secret/secret_manager.go index a339694138b..c7b5b109b2d 100644 --- a/pkg/kubelet/secret/secret_manager.go +++ b/pkg/kubelet/secret/secret_manager.go @@ -23,12 +23,14 @@ import ( "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + corev1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/kubernetes/pkg/kubelet/util/manager" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/watch" ) type Manager interface { @@ -123,3 +125,25 @@ func NewCachingSecretManager(kubeClient clientset.Interface, getTTL manager.GetO manager: manager.NewCacheBasedManager(secretStore, getSecretNames), } } + +// NewWatchingSecretManager creates a manager that keeps a cache of all secrets +// necessary for registered pods. +// It implements the following logic: +// - whenever a pod is created or updated, we start inidvidual watches for all +// referenced objects that aren't referenced from other registered pods +// - every GetObject() returns a value from local cache propagated via watches +func NewWatchingSecretManager(kubeClient clientset.Interface) Manager { + listSecret := func(namespace string, opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.CoreV1().Secrets(namespace).List(opts) + } + watchSecret := func(namespace string, opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.CoreV1().Secrets(namespace).Watch(opts) + } + newSecret := func() runtime.Object { + return &v1.Secret{} + } + gr := corev1.Resource("secret") + return &secretManager{ + manager: manager.NewWatchBasedManager(listSecret, watchSecret, newSecret, gr, getSecretNames), + } +}