From b469404d97616248808f63aed9409707e1d87739 Mon Sep 17 00:00:00 2001 From: louisgong Date: Wed, 4 Dec 2019 12:58:34 +0800 Subject: [PATCH 1/3] hollow-node use remote CRI --- cmd/kubemark/BUILD | 2 -- cmd/kubemark/hollow-node.go | 9 ------- pkg/kubelet/BUILD | 1 + pkg/kubelet/kubelet.go | 6 +++++ pkg/kubelet/remote/fake/fake_runtime.go | 8 +++++++ pkg/kubemark/BUILD | 2 +- pkg/kubemark/hollow_kubelet.go | 31 +++++++++++++------------ 7 files changed, 32 insertions(+), 27 deletions(-) diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD index 82426b25573..6114b765259 100644 --- a/cmd/kubemark/BUILD +++ b/cmd/kubemark/BUILD @@ -19,8 +19,6 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/dockershim:go_default_library", - "//pkg/kubelet/dockershim/libdocker:go_default_library", "//pkg/kubemark:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/iptables/testing:go_default_library", diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 567a930839a..974aa944c9d 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -43,8 +43,6 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/dockershim" - "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubemark" "k8s.io/kubernetes/pkg/master/ports" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" @@ -195,18 +193,11 @@ func run(config *hollowNodeConfig) { } containerManager := cm.NewStubContainerManager() - fakeDockerClientConfig := &dockershim.ClientConfig{ - DockerEndpoint: libdocker.FakeDockerEndpoint, - EnableSleep: true, - WithTraceDisabled: true, - } - hollowKubelet := kubemark.NewHollowKubelet( f, c, client, heartbeatClient, cadvisorInterface, - fakeDockerClientConfig, containerManager, ) hollowKubelet.Run() diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index f942cf4c2aa..b261b5d36da 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -77,6 +77,7 @@ go_library( "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/prober/results:go_default_library", "//pkg/kubelet/remote:go_default_library", + "//pkg/kubelet/remote/fake:go_default_library", "//pkg/kubelet/runtimeclass:go_default_library", "//pkg/kubelet/secret:go_default_library", "//pkg/kubelet/server:go_default_library", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index f77ad998876..105b3ccb4a1 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -93,6 +93,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/prober" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/kubelet/remote" + fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/server" @@ -318,6 +319,11 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku } func getRuntimeAndImageServices(remoteRuntimeEndpoint string, remoteImageEndpoint string, runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) { + if remoteRuntimeEndpoint == fakeremote.FakeRemoteRuntimeEndpoint && remoteImageEndpoint == fakeremote.FakeRemoteImageEndpoint { + fakeRuntime := fakeremote.NewFakeRemoteRuntime() + return fakeRuntime.RuntimeService, fakeRuntime.ImageService, nil + } + rs, err := remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, runtimeRequestTimeout.Duration) if err != nil { return nil, nil, err diff --git a/pkg/kubelet/remote/fake/fake_runtime.go b/pkg/kubelet/remote/fake/fake_runtime.go index af77239ed11..ea6a80b6141 100644 --- a/pkg/kubelet/remote/fake/fake_runtime.go +++ b/pkg/kubelet/remote/fake/fake_runtime.go @@ -28,6 +28,14 @@ import ( utilexec "k8s.io/utils/exec" ) +const ( + // FakeRemoteRuntimeEndpoint used by kubemark hollow-node + FakeRemoteRuntimeEndpoint = "fake://runtime" + + // FakeRemoteImageEndpoint used by kubemark hollow-node + FakeRemoteImageEndpoint = "fake://image" +) + // RemoteRuntime represents a fake remote container runtime. type RemoteRuntime struct { server *grpc.Server diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index 8db94cf6a0e..201a779b054 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -22,7 +22,7 @@ go_library( "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container/testing:go_default_library", - "//pkg/kubelet/dockershim:go_default_library", + "//pkg/kubelet/remote/fake:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/config:go_default_library", diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 83152805560..1be0fff4016 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -32,7 +32,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" - "k8s.io/kubernetes/pkg/kubelet/dockershim" + fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" @@ -98,22 +98,20 @@ func NewHollowKubelet( client *clientset.Clientset, heartbeatClient *clientset.Clientset, cadvisorInterface cadvisor.Interface, - dockerClientConfig *dockershim.ClientConfig, containerManager cm.ContainerManager) *HollowKubelet { d := &kubelet.Dependencies{ - KubeClient: client, - HeartbeatClient: heartbeatClient, - DockerClientConfig: dockerClientConfig, - CAdvisorInterface: cadvisorInterface, - Cloud: nil, - OSInterface: &containertest.FakeOS{}, - ContainerManager: containerManager, - VolumePlugins: volumePlugins(), - TLSOptions: nil, - OOMAdjuster: oom.NewFakeOOMAdjuster(), - Mounter: &mount.FakeMounter{}, - Subpather: &subpath.FakeSubpath{}, - HostUtil: hostutil.NewFakeHostUtil(nil), + KubeClient: client, + HeartbeatClient: heartbeatClient, + CAdvisorInterface: cadvisorInterface, + Cloud: nil, + OSInterface: &containertest.FakeOS{}, + ContainerManager: containerManager, + VolumePlugins: volumePlugins(), + TLSOptions: nil, + OOMAdjuster: oom.NewFakeOOMAdjuster(), + Mounter: &mount.FakeMounter{}, + Subpather: &subpath.FakeSubpath{}, + HostUtil: hostutil.NewFakeHostUtil(nil), } return &HollowKubelet{ @@ -160,6 +158,9 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k f.MaxContainerCount = 100 f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels + f.ContainerRuntimeOptions.ContainerRuntime = kubetypes.RemoteContainerRuntime + f.RemoteRuntimeEndpoint = fakeremote.FakeRemoteRuntimeEndpoint + f.RemoteImageEndpoint = fakeremote.FakeRemoteImageEndpoint f.RegisterNode = true f.RegisterSchedulable = true f.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName) From 0dd468039da91ef87de05505ca9012b36d96a7b7 Mon Sep 17 00:00:00 2001 From: louisgong Date: Fri, 6 Dec 2019 13:17:27 +0800 Subject: [PATCH 2/3] inject remoteRuntime to kubelet dependency --- cmd/kubemark/BUILD | 1 + cmd/kubemark/hollow-node.go | 3 ++ pkg/kubelet/BUILD | 1 - pkg/kubelet/dockershim/docker_service.go | 2 -- pkg/kubelet/dockershim/libdocker/client.go | 14 +-------- pkg/kubelet/kubelet.go | 36 +++++++++------------- pkg/kubelet/remote/fake/fake_runtime.go | 8 ----- pkg/kubemark/hollow_kubelet.go | 29 ++++++++--------- 8 files changed, 35 insertions(+), 59 deletions(-) diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD index 6114b765259..39d3f3a65a9 100644 --- a/cmd/kubemark/BUILD +++ b/cmd/kubemark/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/cm:go_default_library", + "//pkg/kubelet/remote/fake:go_default_library", "//pkg/kubemark:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/iptables/testing:go_default_library", diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 974aa944c9d..0198536eed5 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -43,6 +43,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" + fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" "k8s.io/kubernetes/pkg/kubemark" "k8s.io/kubernetes/pkg/master/ports" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" @@ -192,12 +193,14 @@ func run(config *hollowNodeConfig) { NodeName: config.NodeName, } containerManager := cm.NewStubContainerManager() + fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime() hollowKubelet := kubemark.NewHollowKubelet( f, c, client, heartbeatClient, cadvisorInterface, + fakeRemoteRuntime, containerManager, ) hollowKubelet.Run() diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index b261b5d36da..f942cf4c2aa 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -77,7 +77,6 @@ go_library( "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/prober/results:go_default_library", "//pkg/kubelet/remote:go_default_library", - "//pkg/kubelet/remote/fake:go_default_library", "//pkg/kubelet/runtimeclass:go_default_library", "//pkg/kubelet/secret:go_default_library", "//pkg/kubelet/server:go_default_library", diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 0f07a4ca236..227bef8211a 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -179,8 +179,6 @@ func NewDockerClientFromConfig(config *ClientConfig) libdocker.Interface { config.DockerEndpoint, config.RuntimeRequestTimeout, config.ImagePullProgressDeadline, - config.WithTraceDisabled, - config.EnableSleep, ) return client } diff --git a/pkg/kubelet/dockershim/libdocker/client.go b/pkg/kubelet/dockershim/libdocker/client.go index 97cf509a9e4..72af28d05d2 100644 --- a/pkg/kubelet/dockershim/libdocker/client.go +++ b/pkg/kubelet/dockershim/libdocker/client.go @@ -84,19 +84,7 @@ func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) { // is the timeout for docker requests. If timeout is exceeded, the request // will be cancelled and throw out an error. If requestTimeout is 0, a default // value will be applied. -func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration, - withTraceDisabled bool, enableSleep bool) Interface { - if dockerEndpoint == FakeDockerEndpoint { - fakeClient := NewFakeDockerClient() - if withTraceDisabled { - fakeClient = fakeClient.WithTraceDisabled() - } - - if enableSleep { - fakeClient.EnableSleep = true - } - return fakeClient - } +func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration) Interface { client, err := getDockerClient(dockerEndpoint) if err != nil { klog.Fatalf("Couldn't connect to docker: %v", err) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 105b3ccb4a1..1e840a5394f 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -93,7 +93,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/prober" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/kubelet/remote" - fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/server" @@ -265,6 +264,8 @@ type Dependencies struct { DynamicPluginProber volume.DynamicPluginProber TLSOptions *server.TLSOptions KubeletConfigController *kubeletconfig.Controller + RemoteRuntimeService internalapi.RuntimeService + RemoteImageService internalapi.ImageManagerService } // makePodSourceConfig creates a config.PodConfig from the given @@ -318,23 +319,6 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku return cfg, nil } -func getRuntimeAndImageServices(remoteRuntimeEndpoint string, remoteImageEndpoint string, runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) { - if remoteRuntimeEndpoint == fakeremote.FakeRemoteRuntimeEndpoint && remoteImageEndpoint == fakeremote.FakeRemoteImageEndpoint { - fakeRuntime := fakeremote.NewFakeRemoteRuntime() - return fakeRuntime.RuntimeService, fakeRuntime.ImageService, nil - } - - rs, err := remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, runtimeRequestTimeout.Duration) - if err != nil { - return nil, nil, err - } - is, err := remote.NewRemoteImageService(remoteImageEndpoint, runtimeRequestTimeout.Duration) - if err != nil { - return nil, nil, err - } - return rs, is, err -} - // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules. // No initialization of Kubelet and its modules should happen here. func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, @@ -666,9 +650,19 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, default: return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) } - runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout) - if err != nil { - return nil, err + + runtimeService, imageService := kubeDeps.RemoteRuntimeService, kubeDeps.RemoteImageService + if runtimeService == nil { + runtimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration) + if err != nil { + return nil, err + } + } + if imageService == nil { + imageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration) + if err != nil { + return nil, err + } } klet.runtimeService = runtimeService diff --git a/pkg/kubelet/remote/fake/fake_runtime.go b/pkg/kubelet/remote/fake/fake_runtime.go index ea6a80b6141..af77239ed11 100644 --- a/pkg/kubelet/remote/fake/fake_runtime.go +++ b/pkg/kubelet/remote/fake/fake_runtime.go @@ -28,14 +28,6 @@ import ( utilexec "k8s.io/utils/exec" ) -const ( - // FakeRemoteRuntimeEndpoint used by kubemark hollow-node - FakeRemoteRuntimeEndpoint = "fake://runtime" - - // FakeRemoteImageEndpoint used by kubemark hollow-node - FakeRemoteImageEndpoint = "fake://image" -) - // RemoteRuntime represents a fake remote container runtime. type RemoteRuntime struct { server *grpc.Server diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 1be0fff4016..caceaf097e8 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -98,20 +98,23 @@ func NewHollowKubelet( client *clientset.Clientset, heartbeatClient *clientset.Clientset, cadvisorInterface cadvisor.Interface, + remoteRuntime *fakeremote.RemoteRuntime, containerManager cm.ContainerManager) *HollowKubelet { d := &kubelet.Dependencies{ - KubeClient: client, - HeartbeatClient: heartbeatClient, - CAdvisorInterface: cadvisorInterface, - Cloud: nil, - OSInterface: &containertest.FakeOS{}, - ContainerManager: containerManager, - VolumePlugins: volumePlugins(), - TLSOptions: nil, - OOMAdjuster: oom.NewFakeOOMAdjuster(), - Mounter: &mount.FakeMounter{}, - Subpather: &subpath.FakeSubpath{}, - HostUtil: hostutil.NewFakeHostUtil(nil), + KubeClient: client, + HeartbeatClient: heartbeatClient, + RemoteRuntimeService: remoteRuntime.RuntimeService, + RemoteImageService: remoteRuntime.ImageService, + CAdvisorInterface: cadvisorInterface, + Cloud: nil, + OSInterface: &containertest.FakeOS{}, + ContainerManager: containerManager, + VolumePlugins: volumePlugins(), + TLSOptions: nil, + OOMAdjuster: oom.NewFakeOOMAdjuster(), + Mounter: &mount.FakeMounter{}, + Subpather: &subpath.FakeSubpath{}, + HostUtil: hostutil.NewFakeHostUtil(nil), } return &HollowKubelet{ @@ -159,8 +162,6 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels f.ContainerRuntimeOptions.ContainerRuntime = kubetypes.RemoteContainerRuntime - f.RemoteRuntimeEndpoint = fakeremote.FakeRemoteRuntimeEndpoint - f.RemoteImageEndpoint = fakeremote.FakeRemoteImageEndpoint f.RegisterNode = true f.RegisterSchedulable = true f.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName) From e8e1cc9ee01afa5ff4d45d6a7317fd90c8dc2058 Mon Sep 17 00:00:00 2001 From: louisgong Date: Sat, 14 Dec 2019 13:26:01 +0800 Subject: [PATCH 3/3] extract PreInitRuntimeService from NewMainKubelet --- cmd/kubelet/app/server.go | 23 +++-- pkg/kubelet/kubelet.go | 175 ++++++++++++++++++++------------------ 2 files changed, 102 insertions(+), 96 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 8a705bb139a..ba32570fd2f 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -757,6 +757,17 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f klog.Warning(err) } + err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, + kubeDeps, &s.ContainerRuntimeOptions, + s.ContainerRuntime, + s.RuntimeCgroups, + s.RemoteRuntimeEndpoint, + s.RemoteImageEndpoint, + s.NonMasqueradeCIDR) + if err != nil { + return err + } + if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil { return err } @@ -1062,7 +1073,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeDeps, &kubeServer.ContainerRuntimeOptions, kubeServer.ContainerRuntime, - kubeServer.RuntimeCgroups, kubeServer.HostnameOverride, kubeServer.NodeIP, kubeServer.ProviderID, @@ -1072,8 +1082,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeServer.RegisterNode, kubeServer.RegisterWithTaints, kubeServer.AllowedUnsafeSysctls, - kubeServer.RemoteRuntimeEndpoint, - kubeServer.RemoteImageEndpoint, kubeServer.ExperimentalMounterPath, kubeServer.ExperimentalKernelMemcgNotification, kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount, @@ -1083,7 +1091,6 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeServer.MaxContainerCount, kubeServer.MasterServiceNamespace, kubeServer.RegisterSchedulable, - kubeServer.NonMasqueradeCIDR, kubeServer.KeepTerminatedPodVolumes, kubeServer.NodeLabels, kubeServer.SeccompProfileRoot, @@ -1138,7 +1145,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, crOptions *config.ContainerRuntimeOptions, containerRuntime string, - runtimeCgroups string, hostnameOverride string, nodeIP string, providerID string, @@ -1148,8 +1154,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode bool, registerWithTaints []api.Taint, allowedUnsafeSysctls []string, - remoteRuntimeEndpoint string, - remoteImageEndpoint string, experimentalMounterPath string, experimentalKernelMemcgNotification bool, experimentalCheckNodeCapabilitiesBeforeMount bool, @@ -1159,7 +1163,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount int32, masterServiceNamespace string, registerSchedulable bool, - nonMasqueradeCIDR string, keepTerminatedPodVolumes bool, nodeLabels map[string]string, seccompProfileRoot string, @@ -1172,7 +1175,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps, crOptions, containerRuntime, - runtimeCgroups, hostnameOverride, nodeIP, providerID, @@ -1182,8 +1184,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode, registerWithTaints, allowedUnsafeSysctls, - remoteRuntimeEndpoint, - remoteImageEndpoint, experimentalMounterPath, experimentalKernelMemcgNotification, experimentalCheckNodeCapabilitiesBeforeMount, @@ -1193,7 +1193,6 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount, masterServiceNamespace, registerSchedulable, - nonMasqueradeCIDR, keepTerminatedPodVolumes, nodeLabels, seccompProfileRoot, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 1e840a5394f..c750d036676 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -266,6 +266,10 @@ type Dependencies struct { KubeletConfigController *kubeletconfig.Controller RemoteRuntimeService internalapi.RuntimeService RemoteImageService internalapi.ImageManagerService + criHandler http.Handler + dockerLegacyService dockershim.DockerLegacyService + // remove it after cadvisor.UsingLegacyCadvisorStats dropped. + useLegacyCadvisorStats bool } // makePodSourceConfig creates a config.PodConfig from the given @@ -319,13 +323,90 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku return cfg, nil } +// PreInitRuntimeService will init runtime service before RunKubelet. +func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, + kubeDeps *Dependencies, + crOptions *config.ContainerRuntimeOptions, + containerRuntime string, + runtimeCgroups string, + remoteRuntimeEndpoint string, + remoteImageEndpoint string, + nonMasqueradeCIDR string) error { + if remoteRuntimeEndpoint != "" { + // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified + if remoteImageEndpoint == "" { + remoteImageEndpoint = remoteRuntimeEndpoint + } + } + + switch containerRuntime { + case kubetypes.DockerContainerRuntime: + // TODO: These need to become arguments to a standalone docker shim. + pluginSettings := dockershim.NetworkPluginSettings{ + HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode), + NonMasqueradeCIDR: nonMasqueradeCIDR, + PluginName: crOptions.NetworkPluginName, + PluginConfDir: crOptions.CNIConfDir, + PluginBinDirString: crOptions.CNIBinDir, + PluginCacheDir: crOptions.CNICacheDir, + MTU: int(crOptions.NetworkPluginMTU), + } + + // Create and start the CRI shim running as a grpc server. + streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions) + ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, + &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming) + if err != nil { + return err + } + if crOptions.RedirectContainerStreaming { + kubeDeps.criHandler = ds + } + + // The unix socket for kubelet <-> dockershim communication, dockershim start before runtime service init. + klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", + remoteRuntimeEndpoint, + remoteImageEndpoint) + klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + dockerServer := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) + if err := dockerServer.Start(); err != nil { + return err + } + + // Create dockerLegacyService when the logging driver is not supported. + supported, err := ds.IsCRISupportedLogDriver() + if err != nil { + return err + } + if !supported { + kubeDeps.dockerLegacyService = ds + } + case kubetypes.RemoteContainerRuntime: + // No-op. + break + default: + return fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) + } + + var err error + if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil { + return err + } + if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil { + return err + } + + kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) + + return nil +} + // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules. // No initialization of Kubelet and its modules should happen here. func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies, crOptions *config.ContainerRuntimeOptions, containerRuntime string, - runtimeCgroups string, hostnameOverride string, nodeIP string, providerID string, @@ -335,8 +416,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode bool, registerWithTaints []api.Taint, allowedUnsafeSysctls []string, - remoteRuntimeEndpoint string, - remoteImageEndpoint string, experimentalMounterPath string, experimentalKernelMemcgNotification bool, experimentalCheckNodeCapabilitiesBeforeMount bool, @@ -346,7 +425,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount int32, masterServiceNamespace string, registerSchedulable bool, - nonMasqueradeCIDR string, keepTerminatedPodVolumes bool, nodeLabels map[string]string, seccompProfileRoot string, @@ -589,82 +667,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) - if remoteRuntimeEndpoint != "" { - // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified - if remoteImageEndpoint == "" { - remoteImageEndpoint = remoteRuntimeEndpoint - } - } - - // TODO: These need to become arguments to a standalone docker shim. - pluginSettings := dockershim.NetworkPluginSettings{ - HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode), - NonMasqueradeCIDR: nonMasqueradeCIDR, - PluginName: crOptions.NetworkPluginName, - PluginConfDir: crOptions.CNIConfDir, - PluginBinDirString: crOptions.CNIBinDir, - PluginCacheDir: crOptions.CNICacheDir, - MTU: int(crOptions.NetworkPluginMTU), - } - klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration) - // if left at nil, that means it is unneeded - var legacyLogProvider kuberuntime.LegacyLogProvider - - switch containerRuntime { - case kubetypes.DockerContainerRuntime: - // Create and start the CRI shim running as a grpc server. - streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions) - ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, - &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming) - if err != nil { - return nil, err - } - if crOptions.RedirectContainerStreaming { - klet.criHandler = ds - } - - // The unix socket for kubelet <-> dockershim communication. - klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", - remoteRuntimeEndpoint, - remoteImageEndpoint) - klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") - server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) - if err := server.Start(); err != nil { - return nil, err - } - - // Create dockerLegacyService when the logging driver is not supported. - supported, err := ds.IsCRISupportedLogDriver() - if err != nil { - return nil, err - } - if !supported { - klet.dockerLegacyService = ds - legacyLogProvider = ds - } - case kubetypes.RemoteContainerRuntime: - // No-op. - break - default: - return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) - } - - runtimeService, imageService := kubeDeps.RemoteRuntimeService, kubeDeps.RemoteImageService - if runtimeService == nil { - runtimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration) - if err != nil { - return nil, err - } - } - if imageService == nil { - imageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration) - if err != nil { - return nil, err - } - } - klet.runtimeService = runtimeService + klet.dockerLegacyService = kubeDeps.dockerLegacyService + klet.criHandler = kubeDeps.criHandler + klet.runtimeService = kubeDeps.RemoteRuntimeService if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.KubeClient != nil { klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient) @@ -687,10 +694,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, int(kubeCfg.RegistryBurst), kubeCfg.CPUCFSQuota, kubeCfg.CPUCFSQuotaPeriod, - runtimeService, - imageService, + kubeDeps.RemoteRuntimeService, + kubeDeps.RemoteImageService, kubeDeps.ContainerManager.InternalContainerLifecycle(), - legacyLogProvider, + kubeDeps.dockerLegacyService, klet.runtimeClassManager, ) if err != nil { @@ -706,7 +713,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.runtimeCache = runtimeCache - if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) { + if kubeDeps.useLegacyCadvisorStats { klet.StatsProvider = stats.NewCadvisorStatsProvider( klet.cadvisor, klet.resourceAnalyzer, @@ -720,8 +727,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.resourceAnalyzer, klet.podManager, klet.runtimeCache, - runtimeService, - imageService, + kubeDeps.RemoteRuntimeService, + kubeDeps.RemoteImageService, stats.NewLogMetricsService(), kubecontainer.RealOS{}) }