github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 19:31:44 +00:00
Code Issues Projects Releases Wiki Activity

Revert pull request #85879 "hollow-node use remote CRI"

Browse Source
This commit is contained in:
Jacek Kaniuk 2019-12-19 10:51:48 +01:00
parent 6c98d5515b
commit 4303be3d9f
8 changed files with 136 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/kubelet/app/server.go
View File

@ -757,17 +757,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f
klog.Warning(err)
}
err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration,
kubeDeps, &s.ContainerRuntimeOptions,
s.ContainerRuntime,
s.RuntimeCgroups,
s.RemoteRuntimeEndpoint,
s.RemoteImageEndpoint,
s.NonMasqueradeCIDR)
if err != nil {
return err
}
if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil {
return err
}
@ -1073,6 +1062,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
kubeDeps,
&kubeServer.ContainerRuntimeOptions,
kubeServer.ContainerRuntime,
kubeServer.RuntimeCgroups,
kubeServer.HostnameOverride,
kubeServer.NodeIP,
kubeServer.ProviderID,
@ -1082,6 +1072,8 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
kubeServer.RegisterNode,
kubeServer.RegisterWithTaints,
kubeServer.AllowedUnsafeSysctls,
kubeServer.RemoteRuntimeEndpoint,
kubeServer.RemoteImageEndpoint,
kubeServer.ExperimentalMounterPath,
kubeServer.ExperimentalKernelMemcgNotification,
kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount,
@ -1091,6 +1083,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
kubeServer.MaxContainerCount,
kubeServer.MasterServiceNamespace,
kubeServer.RegisterSchedulable,
kubeServer.NonMasqueradeCIDR,
kubeServer.KeepTerminatedPodVolumes,
kubeServer.NodeLabels,
kubeServer.SeccompProfileRoot,
@ -1145,6 +1138,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *kubelet.Dependencies,
crOptions *config.ContainerRuntimeOptions,
containerRuntime string,
runtimeCgroups string,
hostnameOverride string,
nodeIP string,
providerID string,
@ -1154,6 +1148,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
registerNode bool,
registerWithTaints []api.Taint,
allowedUnsafeSysctls []string,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
experimentalMounterPath string,
experimentalKernelMemcgNotification bool,
experimentalCheckNodeCapabilitiesBeforeMount bool,
@ -1163,6 +1159,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
nonMasqueradeCIDR string,
keepTerminatedPodVolumes bool,
nodeLabels map[string]string,
seccompProfileRoot string,
@ -1175,6 +1172,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps,
crOptions,
containerRuntime,
runtimeCgroups,
hostnameOverride,
nodeIP,
providerID,
@ -1184,6 +1182,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
registerNode,
registerWithTaints,
allowedUnsafeSysctls,
remoteRuntimeEndpoint,
remoteImageEndpoint,
experimentalMounterPath,
experimentalKernelMemcgNotification,
experimentalCheckNodeCapabilitiesBeforeMount,
@ -1193,6 +1193,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
maxContainerCount,
masterServiceNamespace,
registerSchedulable,
nonMasqueradeCIDR,
keepTerminatedPodVolumes,
nodeLabels,
seccompProfileRoot,

3
cmd/kubemark/BUILD
View File

@ -19,7 +19,8 @@ go_library(
"//pkg/api/legacyscheme:go_default_library",
"//pkg/kubelet/cadvisor/testing:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/remote/fake:go_default_library",
"//pkg/kubelet/dockershim:go_default_library",
"//pkg/kubelet/dockershim/libdocker:go_default_library",
"//pkg/kubemark:go_default_library",
"//pkg/master/ports:go_default_library",
"//pkg/util/iptables/testing:go_default_library",

12
cmd/kubemark/hollow-node.go
View File

@ -43,7 +43,8 @@ import (
"k8s.io/kubernetes/pkg/api/legacyscheme"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
"k8s.io/kubernetes/pkg/kubelet/cm"
fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake"
"k8s.io/kubernetes/pkg/kubelet/dockershim"
"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
"k8s.io/kubernetes/pkg/kubemark"
"k8s.io/kubernetes/pkg/master/ports"
fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
@ -193,14 +194,19 @@ func run(config *hollowNodeConfig) {
NodeName: config.NodeName,
}
containerManager := cm.NewStubContainerManager()
fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
fakeDockerClientConfig := &dockershim.ClientConfig{
DockerEndpoint: libdocker.FakeDockerEndpoint,
EnableSleep: true,
WithTraceDisabled: true,
}
hollowKubelet := kubemark.NewHollowKubelet(
f, c,
client,
heartbeatClient,
cadvisorInterface,
fakeRemoteRuntime,
fakeDockerClientConfig,
containerManager,
)
hollowKubelet.Run()

2
pkg/kubelet/dockershim/docker_service.go
View File

@ -179,6 +179,8 @@ func NewDockerClientFromConfig(config *ClientConfig) libdocker.Interface {
config.DockerEndpoint,
config.RuntimeRequestTimeout,
config.ImagePullProgressDeadline,
config.WithTraceDisabled,
config.EnableSleep,
)
return client
}

14
pkg/kubelet/dockershim/libdocker/client.go
View File

@ -84,7 +84,19 @@ func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) {
// is the timeout for docker requests. If timeout is exceeded, the request
// will be cancelled and throw out an error. If requestTimeout is 0, a default
// value will be applied.
func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration) Interface {
func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration,
withTraceDisabled bool, enableSleep bool) Interface {
if dockerEndpoint == FakeDockerEndpoint {
fakeClient := NewFakeDockerClient()
if withTraceDisabled {
fakeClient = fakeClient.WithTraceDisabled()
}
if enableSleep {
fakeClient.EnableSleep = true
}
return fakeClient
}
client, err := getDockerClient(dockerEndpoint)
if err != nil {
klog.Fatalf("Couldn't connect to docker: %v", err)

171
pkg/kubelet/kubelet.go
View File

@ -264,12 +264,6 @@ type Dependencies struct {
DynamicPluginProber volume.DynamicPluginProber
TLSOptions *server.TLSOptions
KubeletConfigController *kubeletconfig.Controller
RemoteRuntimeService internalapi.RuntimeService
RemoteImageService internalapi.ImageManagerService
criHandler http.Handler
dockerLegacyService dockershim.DockerLegacyService
// remove it after cadvisor.UsingLegacyCadvisorStats dropped.
useLegacyCadvisorStats bool
}
// makePodSourceConfig creates a config.PodConfig from the given
@ -323,82 +317,16 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku
return cfg, nil
}
// PreInitRuntimeService will init runtime service before RunKubelet.
func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *Dependencies,
crOptions *config.ContainerRuntimeOptions,
containerRuntime string,
runtimeCgroups string,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
nonMasqueradeCIDR string) error {
if remoteRuntimeEndpoint != "" {
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
if remoteImageEndpoint == "" {
remoteImageEndpoint = remoteRuntimeEndpoint
}
func getRuntimeAndImageServices(remoteRuntimeEndpoint string, remoteImageEndpoint string, runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) {
rs, err := remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, runtimeRequestTimeout.Duration)
if err != nil {
return nil, nil, err
}
switch containerRuntime {
case kubetypes.DockerContainerRuntime:
// TODO: These need to become arguments to a standalone docker shim.
pluginSettings := dockershim.NetworkPluginSettings{
HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode),
NonMasqueradeCIDR: nonMasqueradeCIDR,
PluginName: crOptions.NetworkPluginName,
PluginConfDir: crOptions.CNIConfDir,
PluginBinDirString: crOptions.CNIBinDir,
PluginCacheDir: crOptions.CNICacheDir,
MTU: int(crOptions.NetworkPluginMTU),
}
// Create and start the CRI shim running as a grpc server.
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
if err != nil {
return err
}
if crOptions.RedirectContainerStreaming {
kubeDeps.criHandler = ds
}
// The unix socket for kubelet <-> dockershim communication, dockershim start before runtime service init.
klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
remoteRuntimeEndpoint,
remoteImageEndpoint)
klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
dockerServer := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
if err := dockerServer.Start(); err != nil {
return err
}
// Create dockerLegacyService when the logging driver is not supported.
supported, err := ds.IsCRISupportedLogDriver()
if err != nil {
return err
}
if !supported {
kubeDeps.dockerLegacyService = ds
}
case kubetypes.RemoteContainerRuntime:
// No-op.
break
default:
return fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
is, err := remote.NewRemoteImageService(remoteImageEndpoint, runtimeRequestTimeout.Duration)
if err != nil {
return nil, nil, err
}
var err error
if kubeDeps.RemoteRuntimeService, err = remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil {
return err
}
if kubeDeps.RemoteImageService, err = remote.NewRemoteImageService(remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout.Duration); err != nil {
return err
}
kubeDeps.useLegacyCadvisorStats = cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint)
return nil
return rs, is, err
}
// NewMainKubelet instantiates a new Kubelet object along with all the required internal modules.
@ -407,6 +335,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *Dependencies,
crOptions *config.ContainerRuntimeOptions,
containerRuntime string,
runtimeCgroups string,
hostnameOverride string,
nodeIP string,
providerID string,
@ -416,6 +345,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
registerNode bool,
registerWithTaints []api.Taint,
allowedUnsafeSysctls []string,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
experimentalMounterPath string,
experimentalKernelMemcgNotification bool,
experimentalCheckNodeCapabilitiesBeforeMount bool,
@ -425,6 +356,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
nonMasqueradeCIDR string,
keepTerminatedPodVolumes bool,
nodeLabels map[string]string,
seccompProfileRoot string,
@ -667,11 +599,72 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if remoteRuntimeEndpoint != "" {
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
if remoteImageEndpoint == "" {
remoteImageEndpoint = remoteRuntimeEndpoint
}
}
// TODO: These need to become arguments to a standalone docker shim.
pluginSettings := dockershim.NetworkPluginSettings{
HairpinMode: kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode),
NonMasqueradeCIDR: nonMasqueradeCIDR,
PluginName: crOptions.NetworkPluginName,
PluginConfDir: crOptions.CNIConfDir,
PluginBinDirString: crOptions.CNIBinDir,
PluginCacheDir: crOptions.CNICacheDir,
MTU: int(crOptions.NetworkPluginMTU),
}
klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration)
klet.dockerLegacyService = kubeDeps.dockerLegacyService
klet.criHandler = kubeDeps.criHandler
klet.runtimeService = kubeDeps.RemoteRuntimeService
// if left at nil, that means it is unneeded
var legacyLogProvider kuberuntime.LegacyLogProvider
switch containerRuntime {
case kubetypes.DockerContainerRuntime:
// Create and start the CRI shim running as a grpc server.
streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
if err != nil {
return nil, err
}
if crOptions.RedirectContainerStreaming {
klet.criHandler = ds
}
// The unix socket for kubelet <-> dockershim communication.
klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
remoteRuntimeEndpoint,
remoteImageEndpoint)
klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
if err := server.Start(); err != nil {
return nil, err
}
// Create dockerLegacyService when the logging driver is not supported.
supported, err := ds.IsCRISupportedLogDriver()
if err != nil {
return nil, err
}
if !supported {
klet.dockerLegacyService = ds
legacyLogProvider = ds
}
case kubetypes.RemoteContainerRuntime:
// No-op.
break
default:
return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
}
runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout)
if err != nil {
return nil, err
}
klet.runtimeService = runtimeService
if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.KubeClient != nil {
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
@ -694,10 +687,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
int(kubeCfg.RegistryBurst),
kubeCfg.CPUCFSQuota,
kubeCfg.CPUCFSQuotaPeriod,
kubeDeps.RemoteRuntimeService,
kubeDeps.RemoteImageService,
runtimeService,
imageService,
kubeDeps.ContainerManager.InternalContainerLifecycle(),
kubeDeps.dockerLegacyService,
legacyLogProvider,
klet.runtimeClassManager,
)
if err != nil {
@ -713,7 +706,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
klet.runtimeCache = runtimeCache
if kubeDeps.useLegacyCadvisorStats {
if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) {
klet.StatsProvider = stats.NewCadvisorStatsProvider(
klet.cadvisor,
klet.resourceAnalyzer,
@ -727,8 +720,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.resourceAnalyzer,
klet.podManager,
klet.runtimeCache,
kubeDeps.RemoteRuntimeService,
kubeDeps.RemoteImageService,
runtimeService,
imageService,
stats.NewLogMetricsService(),
kubecontainer.RealOS{})
}

2
pkg/kubemark/BUILD
View File

@ -22,7 +22,7 @@ go_library(
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container/testing:go_default_library",
"//pkg/kubelet/remote/fake:go_default_library",
"//pkg/kubelet/dockershim:go_default_library",
"//pkg/kubelet/types:go_default_library",
"//pkg/proxy:go_default_library",
"//pkg/proxy/config:go_default_library",

32
pkg/kubemark/hollow_kubelet.go
View File

@ -32,7 +32,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/cm"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
fakeremote "k8s.io/kubernetes/pkg/kubelet/remote/fake"
"k8s.io/kubernetes/pkg/kubelet/dockershim"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/volume"
@ -98,23 +98,22 @@ func NewHollowKubelet(
client *clientset.Clientset,
heartbeatClient *clientset.Clientset,
cadvisorInterface cadvisor.Interface,
remoteRuntime *fakeremote.RemoteRuntime,
dockerClientConfig *dockershim.ClientConfig,
containerManager cm.ContainerManager) *HollowKubelet {
d := &kubelet.Dependencies{
KubeClient: client,
HeartbeatClient: heartbeatClient,
RemoteRuntimeService: remoteRuntime.RuntimeService,
RemoteImageService: remoteRuntime.ImageService,
CAdvisorInterface: cadvisorInterface,
Cloud: nil,
OSInterface: &containertest.FakeOS{},
ContainerManager: containerManager,
VolumePlugins: volumePlugins(),
TLSOptions: nil,
OOMAdjuster: oom.NewFakeOOMAdjuster(),
Mounter: &mount.FakeMounter{},
Subpather: &subpath.FakeSubpath{},
HostUtil: hostutil.NewFakeHostUtil(nil),
KubeClient: client,
HeartbeatClient: heartbeatClient,
DockerClientConfig: dockerClientConfig,
CAdvisorInterface: cadvisorInterface,
Cloud: nil,
OSInterface: &containertest.FakeOS{},
ContainerManager: containerManager,
VolumePlugins: volumePlugins(),
TLSOptions: nil,
OOMAdjuster: oom.NewFakeOOMAdjuster(),
Mounter: &mount.FakeMounter{},
Subpather: &subpath.FakeSubpath{},
HostUtil: hostutil.NewFakeHostUtil(nil),
}
return &HollowKubelet{
@ -161,7 +160,6 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k
f.MaxContainerCount = 100
f.MaxPerPodContainerCount = 2
f.NodeLabels = opt.NodeLabels
f.ContainerRuntimeOptions.ContainerRuntime = kubetypes.RemoteContainerRuntime
f.RegisterNode = true
f.RegisterSchedulable = true
f.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)