From 327fc270d75946caef7e21c21c1c7e734cd0803f Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Fri, 7 Apr 2017 16:43:57 -0700 Subject: [PATCH] Add dockershim only mode --- cmd/kubelet/app/BUILD | 3 + cmd/kubelet/app/options/options.go | 2 + cmd/kubelet/app/server.go | 73 +++++++++++++++++++ cmd/kubelet/kubelet.go | 7 ++ hack/verify-flags/known-flags.txt | 1 + pkg/apis/componentconfig/types.go | 3 + pkg/apis/componentconfig/v1alpha1/defaults.go | 3 + pkg/apis/componentconfig/v1alpha1/types.go | 3 + .../v1alpha1/zz_generated.conversion.go | 6 ++ .../v1alpha1/zz_generated.deepcopy.go | 5 ++ pkg/generated/openapi/zz_generated.openapi.go | 7 ++ 11 files changed, 113 insertions(+) diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index c68f6115a52..19aac7b76d9 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -54,6 +54,8 @@ go_library( "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/dockershim:go_default_library", + "//pkg/kubelet/dockershim/remote:go_default_library", "//pkg/kubelet/dockertools:go_default_library", "//pkg/kubelet/eviction:go_default_library", "//pkg/kubelet/eviction/api:go_default_library", @@ -61,6 +63,7 @@ go_library( "//pkg/kubelet/network/cni:go_default_library", "//pkg/kubelet/network/kubenet:go_default_library", "//pkg/kubelet/server:go_default_library", + "//pkg/kubelet/server/streaming:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/csr:go_default_library", "//pkg/util/configz:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 5faed5efba0..75198bb5382 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -296,6 +296,8 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) { // implementation. fs.BoolVar(&c.EnableCRI, "enable-cri", c.EnableCRI, "Enable the Container Runtime Interface (CRI) integration. If --container-runtime is set to \"remote\", Kubelet will communicate with the runtime/image CRI server listening on the endpoint specified by --remote-runtime-endpoint/--remote-image-endpoint. If --container-runtime is set to \"docker\", Kubelet will launch a in-process CRI server on behalf of docker, and communicate over a default endpoint. If --container-runtime is \"rkt\", the flag will be ignored because rkt integration doesn't support CRI yet. [default=true]") fs.MarkDeprecated("enable-cri", "The non-CRI implementation will be deprecated and removed in a future version.") + fs.BoolVar(&c.ExperimentalDockershim, "experimental-dockershim", c.ExperimentalDockershim, "Enable dockershim only mode. In this mode, kubelet will only start dockershim without any other functionalities. This flag only serves test purpose, please do not use it unless you are conscious of what you are doing. [default=false]") + fs.MarkHidden("experimental-dockershim") fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service. The endpoint is used only when CRI integration is enabled (--enable-cri)") fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. The endpoint is used only when CRI integration is enabled (--enable-cri)") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f0a7dc1a073..27b544af919 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -26,6 +26,7 @@ import ( "net" "net/http" _ "net/http/pprof" + "net/url" "os" "path" "strconv" @@ -70,10 +71,13 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockershim" + dockerremote "k8s.io/kubernetes/pkg/kubelet/dockershim/remote" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/eviction" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" "k8s.io/kubernetes/pkg/kubelet/server" + "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/flock" @@ -929,3 +933,72 @@ func parseResourceList(m componentconfig.ConfigurationMap) (v1.ResourceList, err } return rl, nil } + +// RunDockershim only starts the dockershim in current process. This is only used for cri validate testing purpose +// TODO(random-liu): Move this to a separate binary. +func RunDockershim(c *componentconfig.KubeletConfiguration) error { + // Create docker client. + dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, + c.ImagePullProgressDeadline.Duration) + + // Initialize docker exec handler. + var dockerExecHandler dockertools.ExecHandler + switch c.DockerExecHandlerName { + case "native": + dockerExecHandler = &dockertools.NativeExecHandler{} + case "nsenter": + dockerExecHandler = &dockertools.NsenterExecHandler{} + default: + glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName) + dockerExecHandler = &dockertools.NativeExecHandler{} + } + + // Initialize network plugin settings. + binDir := c.CNIBinDir + if binDir == "" { + binDir = c.NetworkPluginDir + } + pluginSettings := dockershim.NetworkPluginSettings{ + HairpinMode: componentconfig.HairpinMode(c.HairpinMode), + NonMasqueradeCIDR: c.NonMasqueradeCIDR, + PluginName: c.NetworkPluginName, + PluginConfDir: c.CNIConfDir, + PluginBinDir: binDir, + MTU: int(c.NetworkPluginMTU), + } + + // Initialize streaming configuration. (Not using TLS now) + streamingConfig := &streaming.Config{ + // Use a relative redirect (no scheme or host). + BaseURL: &url.URL{Path: "/cri/"}, + StreamIdleTimeout: c.StreamingConnectionIdleTimeout.Duration, + StreamCreationTimeout: streaming.DefaultConfig.StreamCreationTimeout, + SupportedRemoteCommandProtocols: streaming.DefaultConfig.SupportedRemoteCommandProtocols, + SupportedPortForwardProtocols: streaming.DefaultConfig.SupportedPortForwardProtocols, + } + + ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage, + streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler) + if err != nil { + return err + } + if err := ds.Start(); err != nil { + return err + } + + // The unix socket for kubelet <-> dockershim communication. + ep := c.RemoteRuntimeEndpoint + if len(ep) == 0 { + ep = "/var/run/dockershim.sock" + } + + glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + server := dockerremote.NewDockerServer(ep, ds) + if err := server.Start(); err != nil { + return err + } + + // Start the streaming server + addr := net.JoinHostPort(c.Address, strconv.Itoa(int(c.Port))) + return http.ListenAndServe(addr, ds) +} diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 47f9617b150..1125e5b926a 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -45,6 +45,13 @@ func main() { verflag.PrintAndExitIfRequested() + if s.ExperimentalDockershim { + if err := app.RunDockershim(&s.KubeletConfiguration); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + } + if err := app.Run(s, nil); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1d83bd6179c..4d7e988b51d 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -236,6 +236,7 @@ experimental-bootstrap-kubeconfig experimental-bootstrap-token-auth experimental-check-node-capabilities-before-mount experimental-cri +experimental-dockershim experimental-fail-swap-on experimental-kernel-memcg-notification experimental-keystone-ca-file diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 78abfe24611..9215a7783f5 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -468,6 +468,9 @@ type KubeletConfiguration struct { // Enable Container Runtime Interface (CRI) integration. // +optional EnableCRI bool + // Enable dockershim only mode. + // +optional + ExperimentalDockershim bool // TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled. // Tells the Kubelet to fail to start if swap is enabled on the node. ExperimentalFailSwapOn bool diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index bb99f8f3cd9..615919c7dc1 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -418,6 +418,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.EnableCRI == nil { obj.EnableCRI = boolVar(true) } + if obj.ExperimentalDockershim == nil { + obj.ExperimentalDockershim = boolVar(false) + } } func boolVar(b bool) *bool { diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 7ef07c29fb8..694d5cceb9e 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -510,6 +510,9 @@ type KubeletConfiguration struct { // Enable Container Runtime Interface (CRI) integration. // +optional EnableCRI *bool `json:"enableCRI,omitempty"` + // Enable dockershim only mode. + // +optional + ExperimentalDockershim *bool `json:"experimentalDockershim,omitempty"` // TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled. // Tells the Kubelet to fail to start if swap is enabled on the node. ExperimentalFailSwapOn bool `json:"experimentalFailSwapOn,omitempty"` diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index 83ae17ffe9b..790622e5565 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -414,6 +414,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu if err := v1.Convert_Pointer_bool_To_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { return err } + if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { + return err + } out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes @@ -611,6 +614,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu if err := v1.Convert_bool_To_Pointer_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { return err } + if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { + return err + } out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index c3d4fbee0dd..f8e5b76a776 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -298,6 +298,11 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * *out = new(bool) **out = **in } + if in.ExperimentalDockershim != nil { + in, out := &in.ExperimentalDockershim, &out.ExperimentalDockershim + *out = new(bool) + **out = **in + } if in.SystemReserved != nil { in, out := &in.SystemReserved, &out.SystemReserved *out = make(map[string]string) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 6f9376dd134..67215bf3df4 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -13785,6 +13785,13 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope Format: "", }, }, + "experimentalDockershim": { + SchemaProps: spec.SchemaProps{ + Description: "Enable dockershim only mode.", + Type: []string{"boolean"}, + Format: "", + }, + }, "experimentalFailSwapOn": { SchemaProps: spec.SchemaProps{ Description: "Tells the Kubelet to fail to start if swap is enabled on the node.",