diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index ba32570fd2f..8a705bb139a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -757,17 +757,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f klog.Warning(err) } - err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, - kubeDeps, &s.ContainerRuntimeOptions, - s.ContainerRuntime, - s.RuntimeCgroups, - s.RemoteRuntimeEndpoint, - s.RemoteImageEndpoint, - s.NonMasqueradeCIDR) - if err != nil { - return err - } - if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil { return err } @@ -1073,6 +1062,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeDeps, &kubeServer.ContainerRuntimeOptions, kubeServer.ContainerRuntime, + kubeServer.RuntimeCgroups, kubeServer.HostnameOverride, kubeServer.NodeIP, kubeServer.ProviderID, @@ -1082,6 +1072,8 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeServer.RegisterNode, kubeServer.RegisterWithTaints, kubeServer.AllowedUnsafeSysctls, + kubeServer.RemoteRuntimeEndpoint, + kubeServer.RemoteImageEndpoint, kubeServer.ExperimentalMounterPath, kubeServer.ExperimentalKernelMemcgNotification, kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount, @@ -1091,6 +1083,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie kubeServer.MaxContainerCount, kubeServer.MasterServiceNamespace, kubeServer.RegisterSchedulable, + kubeServer.NonMasqueradeCIDR, kubeServer.KeepTerminatedPodVolumes, kubeServer.NodeLabels, kubeServer.SeccompProfileRoot, @@ -1145,6 +1138,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, crOptions *config.ContainerRuntimeOptions, containerRuntime string, + runtimeCgroups string, hostnameOverride string, nodeIP string, providerID string, @@ -1154,6 +1148,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode bool, registerWithTaints []api.Taint, allowedUnsafeSysctls []string, + remoteRuntimeEndpoint string, + remoteImageEndpoint string, experimentalMounterPath string, experimentalKernelMemcgNotification bool, experimentalCheckNodeCapabilitiesBeforeMount bool, @@ -1163,6 +1159,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount int32, masterServiceNamespace string, registerSchedulable bool, + nonMasqueradeCIDR string, keepTerminatedPodVolumes bool, nodeLabels map[string]string, seccompProfileRoot string, @@ -1175,6 +1172,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps, crOptions, containerRuntime, + runtimeCgroups, hostnameOverride, nodeIP, providerID, @@ -1184,6 +1182,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode, registerWithTaints, allowedUnsafeSysctls, + remoteRuntimeEndpoint, + remoteImageEndpoint, experimentalMounterPath, experimentalKernelMemcgNotification, experimentalCheckNodeCapabilitiesBeforeMount, @@ -1193,6 +1193,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount, masterServiceNamespace, registerSchedulable, + nonMasqueradeCIDR, keepTerminatedPodVolumes, nodeLabels, seccompProfileRoot, diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD index 39d3f3a65a9..82426b25573 100644 --- a/cmd/kubemark/BUILD +++ b/cmd/kubemark/BUILD @@ -19,7 +19,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/remote/fake:go_default_library", + "//pkg/kubelet/dockershim:go_default_library", + "//pkg/kubelet/dockershim/libdocker:go_default_library", "//pkg/kubemark:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/iptables/testing:go_default_library", diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 0198536eed5..567a930839a 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -43,7 +43,8 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" - fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" + "k8s.io/kubernetes/pkg/kubelet/dockershim" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubemark" "k8s.io/kubernetes/pkg/master/ports" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" @@ -193,14 +194,19 @@ func run(config *hollowNodeConfig) { NodeName: config.NodeName, } containerManager := cm.NewStubContainerManager() - fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime() + + fakeDockerClientConfig := &dockershim.ClientConfig{ + DockerEndpoint: libdocker.FakeDockerEndpoint, + EnableSleep: true, + WithTraceDisabled: true, + } hollowKubelet := kubemark.NewHollowKubelet( f, c, client, heartbeatClient, cadvisorInterface, - fakeRemoteRuntime, + fakeDockerClientConfig, containerManager, ) hollowKubelet.Run() diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 227bef8211a..0f07a4ca236 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -179,6 +179,8 @@ func NewDockerClientFromConfig(config *ClientConfig) libdocker.Interface { config.DockerEndpoint, config.RuntimeRequestTimeout, config.ImagePullProgressDeadline, + config.WithTraceDisabled, + config.EnableSleep, ) return client } diff --git a/pkg/kubelet/dockershim/libdocker/client.go b/pkg/kubelet/dockershim/libdocker/client.go index 72af28d05d2..97cf509a9e4 100644 --- a/pkg/kubelet/dockershim/libdocker/client.go +++ b/pkg/kubelet/dockershim/libdocker/client.go @@ -84,7 +84,19 @@ func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) { // is the timeout for docker requests. If timeout is exceeded, the request // will be cancelled and throw out an error. If requestTimeout is 0, a default // value will be applied. -func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration) Interface { +func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration, + withTraceDisabled bool, enableSleep bool) Interface { + if dockerEndpoint == FakeDockerEndpoint { + fakeClient := NewFakeDockerClient() + if withTraceDisabled { + fakeClient = fakeClient.WithTraceDisabled() + } + + if enableSleep { + fakeClient.EnableSleep = true + } + return fakeClient + } client, err := getDockerClient(dockerEndpoint) if err != nil { klog.Fatalf("Couldn't connect to docker: %v", err) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index c750d036676..f77ad998876 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -264,12 +264,6 @@ type Dependencies struct { DynamicPluginProber volume.DynamicPluginProber TLSOptions *server.TLSOptions KubeletConfigController *kubeletconfig.Controller - RemoteRuntimeService internalapi.RuntimeService - RemoteImageService internalapi.ImageManagerService - criHandler http.Handler - dockerLegacyService dockershim.DockerLegacyService - // remove it after cadvisor.UsingLegacyCadvisorStats dropped. - useLegacyCadvisorStats bool } // makePodSourceConfig creates a config.PodConfig from the given @@ -323,82 +317,16 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku return cfg, nil } -// PreInitRuntimeService will init runtime service before RunKubelet. -func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration, - kubeDeps *Dependencies, - crOptions *config.ContainerRuntimeOptions, - containerRuntime string, - runtimeCgroups string, - remoteRuntimeEndpoint string, - remoteImageEndpoint string, - nonMasqueradeCIDR string) error { - if remoteRuntimeEndpoint != "" { - // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified - if remoteImageEndpoint == "" { - remoteImageEndpoint = remoteRuntimeEndpoint - } +func getRuntimeAndImageServices(remoteRuntimeEndpoint string, remoteImageEndpoint string, runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) { + rs, err := remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, runtimeRequestTimeout.Duration) + if err != nil { + return nil, nil, err } - - switch containerRuntime { - case kubetypes.DockerContainerRuntime: - // TODO: These need to become arguments to a standalone docker shim. - pluginSettings := dockershim.NetworkPluginSettings{ - HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode), - NonMasqueradeCIDR: nonMasqueradeCIDR, - PluginName: crOptions.NetworkPluginName, - PluginConfDir: crOptions.CNIConfDir, - PluginBinDirString: crOptions.CNIBinDir, - PluginCacheDir: crOptions.CNICacheDir, - MTU: int(crOptions.NetworkPluginMTU), - } - - // Create and start the CRI shim running as a grpc server. - streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions) - ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, - &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming) - if err != nil { - return err - } - if crOptions.RedirectContainerStreaming { - kubeDeps.criHandler = ds - } - - // The unix socket for kubelet <-> dockershim communication, dockershim start before runtime service init. - klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", - remoteRuntimeEndpoint, - remoteImageEndpoint) - klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") - dockerServer := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) - if err := dockerServer.Start(); err != nil { - return err - } - - // Create dockerLegacyService when the logging driver is not supported. - supported, err := ds.IsCRISupportedLogDriver() - if err != nil { - return err - } - if !supported { - kubeDeps.dockerLegacyService = ds - } - case kubetypes.RemoteContainerRuntime: - // No-op. - break - default: - return fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) + is, err := remote.NewRemoteImageService(remoteImageEndpoint, runtimeRequestTimeout.Duration) + if err != nil { + return nil, nil, err } - - var err error - if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil { - return err - } - if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil { - return err - } - - kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) - - return nil + return rs, is, err } // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules. @@ -407,6 +335,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies, crOptions *config.ContainerRuntimeOptions, containerRuntime string, + runtimeCgroups string, hostnameOverride string, nodeIP string, providerID string, @@ -416,6 +345,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode bool, registerWithTaints []api.Taint, allowedUnsafeSysctls []string, + remoteRuntimeEndpoint string, + remoteImageEndpoint string, experimentalMounterPath string, experimentalKernelMemcgNotification bool, experimentalCheckNodeCapabilitiesBeforeMount bool, @@ -425,6 +356,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, maxContainerCount int32, masterServiceNamespace string, registerSchedulable bool, + nonMasqueradeCIDR string, keepTerminatedPodVolumes bool, nodeLabels map[string]string, seccompProfileRoot string, @@ -667,11 +599,72 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) + if remoteRuntimeEndpoint != "" { + // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified + if remoteImageEndpoint == "" { + remoteImageEndpoint = remoteRuntimeEndpoint + } + } + + // TODO: These need to become arguments to a standalone docker shim. + pluginSettings := dockershim.NetworkPluginSettings{ + HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode), + NonMasqueradeCIDR: nonMasqueradeCIDR, + PluginName: crOptions.NetworkPluginName, + PluginConfDir: crOptions.CNIConfDir, + PluginBinDirString: crOptions.CNIBinDir, + PluginCacheDir: crOptions.CNICacheDir, + MTU: int(crOptions.NetworkPluginMTU), + } + klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration) - klet.dockerLegacyService = kubeDeps.dockerLegacyService - klet.criHandler = kubeDeps.criHandler - klet.runtimeService = kubeDeps.RemoteRuntimeService + // if left at nil, that means it is unneeded + var legacyLogProvider kuberuntime.LegacyLogProvider + + switch containerRuntime { + case kubetypes.DockerContainerRuntime: + // Create and start the CRI shim running as a grpc server. + streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions) + ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, + &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming) + if err != nil { + return nil, err + } + if crOptions.RedirectContainerStreaming { + klet.criHandler = ds + } + + // The unix socket for kubelet <-> dockershim communication. + klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", + remoteRuntimeEndpoint, + remoteImageEndpoint) + klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) + if err := server.Start(); err != nil { + return nil, err + } + + // Create dockerLegacyService when the logging driver is not supported. + supported, err := ds.IsCRISupportedLogDriver() + if err != nil { + return nil, err + } + if !supported { + klet.dockerLegacyService = ds + legacyLogProvider = ds + } + case kubetypes.RemoteContainerRuntime: + // No-op. + break + default: + return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) + } + runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout) + if err != nil { + return nil, err + } + klet.runtimeService = runtimeService if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.KubeClient != nil { klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient) @@ -694,10 +687,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, int(kubeCfg.RegistryBurst), kubeCfg.CPUCFSQuota, kubeCfg.CPUCFSQuotaPeriod, - kubeDeps.RemoteRuntimeService, - kubeDeps.RemoteImageService, + runtimeService, + imageService, kubeDeps.ContainerManager.InternalContainerLifecycle(), - kubeDeps.dockerLegacyService, + legacyLogProvider, klet.runtimeClassManager, ) if err != nil { @@ -713,7 +706,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.runtimeCache = runtimeCache - if kubeDeps.useLegacyCadvisorStats { + if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) { klet.StatsProvider = stats.NewCadvisorStatsProvider( klet.cadvisor, klet.resourceAnalyzer, @@ -727,8 +720,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.resourceAnalyzer, klet.podManager, klet.runtimeCache, - kubeDeps.RemoteRuntimeService, - kubeDeps.RemoteImageService, + runtimeService, + imageService, stats.NewLogMetricsService(), kubecontainer.RealOS{}) } diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index 201a779b054..8db94cf6a0e 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -22,7 +22,7 @@ go_library( "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container/testing:go_default_library", - "//pkg/kubelet/remote/fake:go_default_library", + "//pkg/kubelet/dockershim:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/config:go_default_library", diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index caceaf097e8..83152805560 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -32,7 +32,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" - fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake" + "k8s.io/kubernetes/pkg/kubelet/dockershim" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" @@ -98,23 +98,22 @@ func NewHollowKubelet( client *clientset.Clientset, heartbeatClient *clientset.Clientset, cadvisorInterface cadvisor.Interface, - remoteRuntime *fakeremote.RemoteRuntime, + dockerClientConfig *dockershim.ClientConfig, containerManager cm.ContainerManager) *HollowKubelet { d := &kubelet.Dependencies{ - KubeClient: client, - HeartbeatClient: heartbeatClient, - RemoteRuntimeService: remoteRuntime.RuntimeService, - RemoteImageService: remoteRuntime.ImageService, - CAdvisorInterface: cadvisorInterface, - Cloud: nil, - OSInterface: &containertest.FakeOS{}, - ContainerManager: containerManager, - VolumePlugins: volumePlugins(), - TLSOptions: nil, - OOMAdjuster: oom.NewFakeOOMAdjuster(), - Mounter: &mount.FakeMounter{}, - Subpather: &subpath.FakeSubpath{}, - HostUtil: hostutil.NewFakeHostUtil(nil), + KubeClient: client, + HeartbeatClient: heartbeatClient, + DockerClientConfig: dockerClientConfig, + CAdvisorInterface: cadvisorInterface, + Cloud: nil, + OSInterface: &containertest.FakeOS{}, + ContainerManager: containerManager, + VolumePlugins: volumePlugins(), + TLSOptions: nil, + OOMAdjuster: oom.NewFakeOOMAdjuster(), + Mounter: &mount.FakeMounter{}, + Subpather: &subpath.FakeSubpath{}, + HostUtil: hostutil.NewFakeHostUtil(nil), } return &HollowKubelet{ @@ -161,7 +160,6 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k f.MaxContainerCount = 100 f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels - f.ContainerRuntimeOptions.ContainerRuntime = kubetypes.RemoteContainerRuntime f.RegisterNode = true f.RegisterSchedulable = true f.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)