diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 610c64bf368..25272a1d77a 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -55,6 +55,8 @@ go_library( "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/dockershim:go_default_library", + "//pkg/kubelet/dockershim/remote:go_default_library", "//pkg/kubelet/dockertools:go_default_library", "//pkg/kubelet/eviction:go_default_library", "//pkg/kubelet/eviction/api:go_default_library", @@ -62,6 +64,7 @@ go_library( "//pkg/kubelet/network/cni:go_default_library", "//pkg/kubelet/network/kubenet:go_default_library", "//pkg/kubelet/server:go_default_library", + "//pkg/kubelet/server/streaming:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/csr:go_default_library", "//pkg/util/configz:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 5faed5efba0..75198bb5382 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -296,6 +296,8 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) { // implementation. fs.BoolVar(&c.EnableCRI, "enable-cri", c.EnableCRI, "Enable the Container Runtime Interface (CRI) integration. If --container-runtime is set to \"remote\", Kubelet will communicate with the runtime/image CRI server listening on the endpoint specified by --remote-runtime-endpoint/--remote-image-endpoint. If --container-runtime is set to \"docker\", Kubelet will launch a in-process CRI server on behalf of docker, and communicate over a default endpoint. If --container-runtime is \"rkt\", the flag will be ignored because rkt integration doesn't support CRI yet. [default=true]") fs.MarkDeprecated("enable-cri", "The non-CRI implementation will be deprecated and removed in a future version.") + fs.BoolVar(&c.ExperimentalDockershim, "experimental-dockershim", c.ExperimentalDockershim, "Enable dockershim only mode. In this mode, kubelet will only start dockershim without any other functionalities. This flag only serves test purpose, please do not use it unless you are conscious of what you are doing. [default=false]") + fs.MarkHidden("experimental-dockershim") fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service. The endpoint is used only when CRI integration is enabled (--enable-cri)") fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. The endpoint is used only when CRI integration is enabled (--enable-cri)") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f0a7dc1a073..27b544af919 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -26,6 +26,7 @@ import ( "net" "net/http" _ "net/http/pprof" + "net/url" "os" "path" "strconv" @@ -70,10 +71,13 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockershim" + dockerremote "k8s.io/kubernetes/pkg/kubelet/dockershim/remote" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/eviction" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" "k8s.io/kubernetes/pkg/kubelet/server" + "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/flock" @@ -929,3 +933,72 @@ func parseResourceList(m componentconfig.ConfigurationMap) (v1.ResourceList, err } return rl, nil } + +// RunDockershim only starts the dockershim in current process. This is only used for cri validate testing purpose +// TODO(random-liu): Move this to a separate binary. +func RunDockershim(c *componentconfig.KubeletConfiguration) error { + // Create docker client. + dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, + c.ImagePullProgressDeadline.Duration) + + // Initialize docker exec handler. + var dockerExecHandler dockertools.ExecHandler + switch c.DockerExecHandlerName { + case "native": + dockerExecHandler = &dockertools.NativeExecHandler{} + case "nsenter": + dockerExecHandler = &dockertools.NsenterExecHandler{} + default: + glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName) + dockerExecHandler = &dockertools.NativeExecHandler{} + } + + // Initialize network plugin settings. + binDir := c.CNIBinDir + if binDir == "" { + binDir = c.NetworkPluginDir + } + pluginSettings := dockershim.NetworkPluginSettings{ + HairpinMode: componentconfig.HairpinMode(c.HairpinMode), + NonMasqueradeCIDR: c.NonMasqueradeCIDR, + PluginName: c.NetworkPluginName, + PluginConfDir: c.CNIConfDir, + PluginBinDir: binDir, + MTU: int(c.NetworkPluginMTU), + } + + // Initialize streaming configuration. (Not using TLS now) + streamingConfig := &streaming.Config{ + // Use a relative redirect (no scheme or host). + BaseURL: &url.URL{Path: "/cri/"}, + StreamIdleTimeout: c.StreamingConnectionIdleTimeout.Duration, + StreamCreationTimeout: streaming.DefaultConfig.StreamCreationTimeout, + SupportedRemoteCommandProtocols: streaming.DefaultConfig.SupportedRemoteCommandProtocols, + SupportedPortForwardProtocols: streaming.DefaultConfig.SupportedPortForwardProtocols, + } + + ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage, + streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler) + if err != nil { + return err + } + if err := ds.Start(); err != nil { + return err + } + + // The unix socket for kubelet <-> dockershim communication. + ep := c.RemoteRuntimeEndpoint + if len(ep) == 0 { + ep = "/var/run/dockershim.sock" + } + + glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + server := dockerremote.NewDockerServer(ep, ds) + if err := server.Start(); err != nil { + return err + } + + // Start the streaming server + addr := net.JoinHostPort(c.Address, strconv.Itoa(int(c.Port))) + return http.ListenAndServe(addr, ds) +} diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 47f9617b150..1125e5b926a 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -45,6 +45,13 @@ func main() { verflag.PrintAndExitIfRequested() + if s.ExperimentalDockershim { + if err := app.RunDockershim(&s.KubeletConfiguration); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + } + if err := app.Run(s, nil); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 2a3c6e36090..82011a3b3dc 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -236,6 +236,7 @@ experimental-bootstrap-kubeconfig experimental-bootstrap-token-auth experimental-check-node-capabilities-before-mount experimental-cri +experimental-dockershim experimental-fail-swap-on experimental-kernel-memcg-notification experimental-keystone-ca-file diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 7b0f935cd79..0ce875d66ec 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -468,6 +468,9 @@ type KubeletConfiguration struct { // Enable Container Runtime Interface (CRI) integration. // +optional EnableCRI bool + // Enable dockershim only mode. + // +optional + ExperimentalDockershim bool // TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled. // Tells the Kubelet to fail to start if swap is enabled on the node. ExperimentalFailSwapOn bool diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index b5c7c3b3433..6a5137f92b5 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -421,6 +421,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.EnableCRI == nil { obj.EnableCRI = boolVar(true) } + if obj.ExperimentalDockershim == nil { + obj.ExperimentalDockershim = boolVar(false) + } } func boolVar(b bool) *bool { diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 48b487c4bad..847168ae707 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -523,6 +523,9 @@ type KubeletConfiguration struct { // Enable Container Runtime Interface (CRI) integration. // +optional EnableCRI *bool `json:"enableCRI,omitempty"` + // Enable dockershim only mode. + // +optional + ExperimentalDockershim *bool `json:"experimentalDockershim,omitempty"` // TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled. // Tells the Kubelet to fail to start if swap is enabled on the node. ExperimentalFailSwapOn bool `json:"experimentalFailSwapOn,omitempty"` diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index 5dfdc9e7b3e..f5912c0b430 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -420,6 +420,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu if err := v1.Convert_Pointer_bool_To_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { return err } + if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { + return err + } out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes @@ -617,6 +620,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu if err := v1.Convert_bool_To_Pointer_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { return err } + if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { + return err + } out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index c3d4fbee0dd..f8e5b76a776 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -298,6 +298,11 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * *out = new(bool) **out = **in } + if in.ExperimentalDockershim != nil { + in, out := &in.ExperimentalDockershim, &out.ExperimentalDockershim + *out = new(bool) + **out = **in + } if in.SystemReserved != nil { in, out := &in.SystemReserved, &out.SystemReserved *out = make(map[string]string) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index f6422ea07c7..fa4624bbf68 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -13964,6 +13964,13 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope Format: "", }, }, + "experimentalDockershim": { + SchemaProps: spec.SchemaProps{ + Description: "Enable dockershim only mode.", + Type: []string{"boolean"}, + Format: "", + }, + }, "experimentalFailSwapOn": { SchemaProps: spec.SchemaProps{ Description: "Tells the Kubelet to fail to start if swap is enabled on the node.",