Merge pull request #45194 from yujuhong/rm-cri-flag

Automatic merge from submit-queue

Remove the deprecated `--enable-cri` flag

Except for rkt, CRI is the default and only integration point for
container runtimes.

```release-note
Remove the deprecated `--enable-cri` flag. CRI is now the default, 
and the only way to integrate with kubelet for the container runtimes.
```

cmd/kubelet/app/options/options.go

@@ -301,19 +301,10 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&c.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", c.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates.  Can be useful for debugging volume related issues.")
 
 	// CRI flags.
-	// TODO: Remove experimental-cri in kubernetes 1.7.
-	fs.BoolVar(&c.EnableCRI, "experimental-cri", c.EnableCRI, "Same as --enable-cri.")
-	fs.MarkDeprecated("experimental-cri", "Please use --enable-cri instead.")
-	fs.MarkHidden("experimental-cri")
-	// TODO: Remove enable-cri once we stop supporting the non-cri
-	// implementation.
-	fs.BoolVar(&c.EnableCRI, "enable-cri", c.EnableCRI, "Enable the Container Runtime Interface (CRI) integration. If --container-runtime is set to \"remote\", Kubelet will communicate with the runtime/image CRI server listening on the endpoint specified by --remote-runtime-endpoint/--remote-image-endpoint. If --container-runtime is set to \"docker\", Kubelet will launch a in-process CRI server on behalf of docker, and communicate over a default endpoint. If --container-runtime is \"rkt\", the flag will be ignored because rkt integration doesn't support CRI yet.")
-	fs.MarkDeprecated("enable-cri", "The non-CRI implementation will be deprecated and removed in a future version.")
 	fs.BoolVar(&c.ExperimentalDockershim, "experimental-dockershim", c.ExperimentalDockershim, "Enable dockershim only mode. In this mode, kubelet will only start dockershim without any other functionalities. This flag only serves test purpose, please do not use it unless you are conscious of what you are doing. [default=false]")
 	fs.MarkHidden("experimental-dockershim")
-
+	fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service.")
-	fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service. The endpoint is used only when CRI integration is enabled (--enable-cri)")
+	fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default.")
-	fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. The endpoint is used only when CRI integration is enabled (--enable-cri)")
 	fs.BoolVar(&c.DockerDisableSharedPID, "docker-disable-shared-pid", c.DockerDisableSharedPID, "The Container Runtime Interface (CRI) defaults to using a shared PID namespace for containers in a pod when running with Docker 1.13.1 or higher. Setting this flag reverts to the previous behavior of isolated PID namespaces. This ability will be removed in a future Kubernetes release.")
 
 	fs.BoolVar(&c.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", c.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required componenets (binaries, etc.) before performing the mount")

cmd/kubelet/app/server.go

@@ -544,7 +544,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 				CgroupRoot:            s.CgroupRoot,
 				CgroupDriver:          s.CgroupDriver,
 				ProtectKernelDefaults: s.ProtectKernelDefaults,
-				EnableCRI:             s.EnableCRI,
 				NodeAllocatableConfig: cm.NodeAllocatableConfig{
 					KubeReservedCgroupName:   s.KubeReservedCgroup,
 					SystemReservedCgroupName: s.SystemReservedCgroup,

hack/local-up-cluster.sh

@@ -622,7 +622,6 @@ function start_kubelet {
       fi
 
       sudo -E "${GO_OUT}/hyperkube" kubelet ${priv_arg}\
-        --enable-cri="${ENABLE_CRI}" \
         --v=${LOG_LEVEL} \
         --chaos-chance="${CHAOS_CHANCE}" \
         --container-runtime="${CONTAINER_RUNTIME}" \

hack/verify-flags/known-flags.txt

@@ -193,7 +193,6 @@ duration-sec
 e2e-output-dir
 e2e-verify-service-account
 enable-controller-attach-detach
-enable-cri
 enable-custom-metrics
 enable-debugging-handlers
 enable-dynamic-provisioning

pkg/apis/componentconfig/types.go

@@ -508,9 +508,6 @@ type KubeletConfiguration struct {
 	// featureGates is a string of comma-separated key=value pairs that describe feature
 	// gates for alpha/experimental features.
 	FeatureGates string
-	// Enable Container Runtime Interface (CRI) integration.
-	// +optional
-	EnableCRI bool
 	// Enable dockershim only mode.
 	// +optional
 	ExperimentalDockershim bool

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -431,9 +431,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.EnforceNodeAllocatable == nil {
 		obj.EnforceNodeAllocatable = defaultNodeAllocatableEnforcement
 	}
-	if obj.EnableCRI == nil {
-		obj.EnableCRI = boolVar(true)
-	}
 	if obj.ExperimentalDockershim == nil {
 		obj.ExperimentalDockershim = boolVar(false)
 	}

pkg/apis/componentconfig/v1alpha1/types.go

@@ -563,9 +563,6 @@ type KubeletConfiguration struct {
 	// featureGates is a string of comma-separated key=value pairs that describe feature
 	// gates for alpha/experimental features.
 	FeatureGates string `json:"featureGates,omitempty"`
-	// Enable Container Runtime Interface (CRI) integration.
-	// +optional
-	EnableCRI *bool `json:"enableCRI,omitempty"`
 	// Enable dockershim only mode.
 	// +optional
 	ExperimentalDockershim *bool `json:"experimentalDockershim,omitempty"`

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -515,9 +515,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	}
 	out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
 	out.FeatureGates = in.FeatureGates
-	if err := v1.Convert_Pointer_bool_To_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil {
-		return err
-	}
 	if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil {
 		return err
 	}
@@ -719,9 +716,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	}
 	out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
 	out.FeatureGates = in.FeatureGates
-	if err := v1.Convert_bool_To_Pointer_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil {
-		return err
-	}
 	if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil {
 		return err
 	}

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -326,11 +326,6 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 			*out = make([]string, len(*in))
 			copy(*out, *in)
 		}
-		if in.EnableCRI != nil {
-			in, out := &in.EnableCRI, &out.EnableCRI
-			*out = new(bool)
-			**out = **in
-		}
 		if in.ExperimentalDockershim != nil {
 			in, out := &in.ExperimentalDockershim, &out.ExperimentalDockershim
 			*out = new(bool)

pkg/kubelet/cm/container_manager.go

@@ -74,7 +74,6 @@ type NodeConfig struct {
 	CgroupRoot            string
 	CgroupDriver          string
 	ProtectKernelDefaults bool
-	EnableCRI             bool
 	NodeAllocatableConfig
 	ExperimentalQOSReserved map[v1.ResourceName]int64
 }

pkg/kubelet/cm/container_manager_linux.go

@@ -377,70 +377,25 @@ func (cm *containerManagerImpl) setupNode(activePods ActivePodsFunc) error {
 
 	systemContainers := []*systemContainer{}
 	if cm.ContainerRuntime == "docker" {
-		dockerAPIVersion := getDockerAPIVersion(cm.cadvisorInterface)
+		// With the docker-CRI integration, dockershim will manage the cgroups
-		if cm.EnableCRI {
+		// and oom score for the docker processes.
-			// If kubelet uses CRI, dockershim will manage the cgroups and oom
+		// In the future, NodeSpec should mandate the cgroup that the
-			// score for the docker processes.
+		// runtime processes need to be in. For now, we still check the
-			// In the future, NodeSpec should mandate the cgroup that the
+		// cgroup for docker periodically, so that kubelet can recognize
-			// runtime processes need to be in. For now, we still check the
+		// the cgroup for docker and serve stats for the runtime.
-			// cgroup for docker periodically, so that kubelet can recognize
+		// TODO(#27097): Fix this after NodeSpec is clearly defined.
-			// the cgroup for docker and serve stats for the runtime.
+		cm.periodicTasks = append(cm.periodicTasks, func() {
-			// TODO(#27097): Fix this after NodeSpec is clearly defined.
+			glog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration")
-			cm.periodicTasks = append(cm.periodicTasks, func() {
+			cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile)
-				glog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration")
+			if err != nil {
-				cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile)
+				glog.Error(err)
-				if err != nil {
+				return
-					glog.Error(err)
-					return
-				}
-				glog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont)
-				cm.Lock()
-				defer cm.Unlock()
-				cm.RuntimeCgroupsName = cont
-			})
-		} else if cm.RuntimeCgroupsName != "" {
-			cont := newSystemCgroups(cm.RuntimeCgroupsName)
-			memoryLimit := (int64(cm.capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100))
-			if memoryLimit < MinDockerMemoryLimit {
-				glog.Warningf("Memory limit %d for container %s is too small, reset it to %d", memoryLimit, cm.RuntimeCgroupsName, MinDockerMemoryLimit)
-				memoryLimit = MinDockerMemoryLimit
 			}
-
+			glog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont)
-			glog.V(2).Infof("Configure resource-only container %s with memory limit: %d", cm.RuntimeCgroupsName, memoryLimit)
+			cm.Lock()
-			allowAllDevices := true
+			defer cm.Unlock()
-			dockerContainer := &fs.Manager{
+			cm.RuntimeCgroupsName = cont
-				Cgroups: &configs.Cgroup{
+		})
-					Parent: "/",
-					Name:   cm.RuntimeCgroupsName,
-					Resources: &configs.Resources{
-						Memory:          memoryLimit,
-						MemorySwap:      -1,
-						AllowAllDevices: &allowAllDevices,
-					},
-				},
-			}
-			cont.ensureStateFunc = func(manager *fs.Manager) error {
-				return EnsureDockerInContainer(dockerAPIVersion, qos.DockerOOMScoreAdj, dockerContainer)
-			}
-			systemContainers = append(systemContainers, cont)
-		} else {
-			cm.periodicTasks = append(cm.periodicTasks, func() {
-				glog.V(10).Infof("Adding docker daemon periodic tasks")
-				if err := EnsureDockerInContainer(dockerAPIVersion, qos.DockerOOMScoreAdj, nil); err != nil {
-					glog.Error(err)
-					return
-				}
-				cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile)
-				if err != nil {
-					glog.Error(err)
-					return
-				}
-				glog.V(2).Infof("Discovered runtime cgroups name: %s", cont)
-				cm.Lock()
-				defer cm.Unlock()
-				cm.RuntimeCgroupsName = cont
-			})
-		}
 	}
 
 	if cm.SystemCgroupsName != "" {

pkg/kubelet/kubelet.go

@@ -530,12 +530,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 	pluginSettings.LegacyRuntimeHost = nl
 
 	// rktnetes cannot be run with CRI.
-	// TODO(yujuhong): Remove the EnableCRI field.
 	if kubeCfg.ContainerRuntime != "rkt" {
-		kubeCfg.EnableCRI = true
-	}
-
-	if kubeCfg.EnableCRI {
 		// kubelet defers to the runtime shim to setup networking. Setting
 		// this to nil will prevent it from trying to invoke the plugin.
 		// It's easier to always probe and initialize plugins till cri
@@ -614,6 +609,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		klet.containerRuntime = runtime
 		klet.runner = runtime
 	} else {
+		// rkt uses the legacy, non-CRI, integration. Configure it the old way.
 		// TODO: Include hairpin mode settings in rkt?
 		conf := &rkt.Config{
 			Path:            kubeCfg.RktPath,
@@ -2018,9 +2014,9 @@ func (kl *Kubelet) updateRuntimeUp() {
 		glog.Errorf("Container runtime sanity check failed: %v", err)
 		return
 	}
-	// Only check specific conditions when runtime integration type is cri,
+	// rkt uses the legacy, non-CRI integration. Don't check the runtime
-	// because the old integration doesn't populate any runtime condition.
+	// conditions for it.
-	if kl.kubeletConfiguration.EnableCRI && kl.kubeletConfiguration.ContainerRuntime != "rkt" {
+	if kl.kubeletConfiguration.ContainerRuntime != "rkt" {
 		if s == nil {
 			glog.Errorf("Container runtime status is nil")
 			return

pkg/kubelet/kubelet_node_status_test.go

@@ -812,8 +812,6 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
 	kubelet.updateRuntimeUp()
 	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")
 
-	// Test cri integration.
-	kubelet.kubeletConfiguration.EnableCRI = true
 	fakeRuntime.StatusErr = nil
 
 	// Should report node not ready if runtime status is nil.

pkg/kubelet/kubelet_test.go

@@ -143,6 +143,14 @@ func newTestKubeletWithImageList(
 	fakeRuntime.RuntimeType = "test"
 	fakeRuntime.VersionInfo = "1.5.0"
 	fakeRuntime.ImageList = imageList
+	// Set ready conditions by default.
+	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
+		Conditions: []kubecontainer.RuntimeCondition{
+			{Type: "RuntimeReady", Status: true},
+			{Type: "NetworkReady", Status: true},
+		},
+	}
+
 	fakeRecorder := &record.FakeRecorder{}
 	fakeKubeClient := &fake.Clientset{}
 	kubelet := &Kubelet{}