diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 513f58b0442..f98d379bb5b 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -301,19 +301,10 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) { fs.BoolVar(&c.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", c.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates. Can be useful for debugging volume related issues.") // CRI flags. - // TODO: Remove experimental-cri in kubernetes 1.7. - fs.BoolVar(&c.EnableCRI, "experimental-cri", c.EnableCRI, "Same as --enable-cri.") - fs.MarkDeprecated("experimental-cri", "Please use --enable-cri instead.") - fs.MarkHidden("experimental-cri") - // TODO: Remove enable-cri once we stop supporting the non-cri - // implementation. - fs.BoolVar(&c.EnableCRI, "enable-cri", c.EnableCRI, "Enable the Container Runtime Interface (CRI) integration. If --container-runtime is set to \"remote\", Kubelet will communicate with the runtime/image CRI server listening on the endpoint specified by --remote-runtime-endpoint/--remote-image-endpoint. If --container-runtime is set to \"docker\", Kubelet will launch a in-process CRI server on behalf of docker, and communicate over a default endpoint. If --container-runtime is \"rkt\", the flag will be ignored because rkt integration doesn't support CRI yet.") - fs.MarkDeprecated("enable-cri", "The non-CRI implementation will be deprecated and removed in a future version.") fs.BoolVar(&c.ExperimentalDockershim, "experimental-dockershim", c.ExperimentalDockershim, "Enable dockershim only mode. In this mode, kubelet will only start dockershim without any other functionalities. This flag only serves test purpose, please do not use it unless you are conscious of what you are doing. [default=false]") fs.MarkHidden("experimental-dockershim") - - fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service. The endpoint is used only when CRI integration is enabled (--enable-cri)") - fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. The endpoint is used only when CRI integration is enabled (--enable-cri)") + fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service.") + fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default.") fs.BoolVar(&c.DockerDisableSharedPID, "docker-disable-shared-pid", c.DockerDisableSharedPID, "The Container Runtime Interface (CRI) defaults to using a shared PID namespace for containers in a pod when running with Docker 1.13.1 or higher. Setting this flag reverts to the previous behavior of isolated PID namespaces. This ability will be removed in a future Kubernetes release.") fs.BoolVar(&c.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", c.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required componenets (binaries, etc.) before performing the mount") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 48351b91b42..d2a6cf31b49 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -544,7 +544,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { CgroupRoot: s.CgroupRoot, CgroupDriver: s.CgroupDriver, ProtectKernelDefaults: s.ProtectKernelDefaults, - EnableCRI: s.EnableCRI, NodeAllocatableConfig: cm.NodeAllocatableConfig{ KubeReservedCgroupName: s.KubeReservedCgroup, SystemReservedCgroupName: s.SystemReservedCgroup, diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index ef6962a29db..fd9f334dfc0 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -622,7 +622,6 @@ function start_kubelet { fi sudo -E "${GO_OUT}/hyperkube" kubelet ${priv_arg}\ - --enable-cri="${ENABLE_CRI}" \ --v=${LOG_LEVEL} \ --chaos-chance="${CHAOS_CHANCE}" \ --container-runtime="${CONTAINER_RUNTIME}" \ diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 96857f0f5ca..7ea167c4219 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -193,7 +193,6 @@ duration-sec e2e-output-dir e2e-verify-service-account enable-controller-attach-detach -enable-cri enable-custom-metrics enable-debugging-handlers enable-dynamic-provisioning diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 1b619003ccd..69cc119424a 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -508,9 +508,6 @@ type KubeletConfiguration struct { // featureGates is a string of comma-separated key=value pairs that describe feature // gates for alpha/experimental features. FeatureGates string - // Enable Container Runtime Interface (CRI) integration. - // +optional - EnableCRI bool // Enable dockershim only mode. // +optional ExperimentalDockershim bool diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 7bd93b8f726..06b01c0df48 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -431,9 +431,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.EnforceNodeAllocatable == nil { obj.EnforceNodeAllocatable = defaultNodeAllocatableEnforcement } - if obj.EnableCRI == nil { - obj.EnableCRI = boolVar(true) - } if obj.ExperimentalDockershim == nil { obj.ExperimentalDockershim = boolVar(false) } diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index bcb6e80d2e4..8211e8f0aaa 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -563,9 +563,6 @@ type KubeletConfiguration struct { // featureGates is a string of comma-separated key=value pairs that describe feature // gates for alpha/experimental features. FeatureGates string `json:"featureGates,omitempty"` - // Enable Container Runtime Interface (CRI) integration. - // +optional - EnableCRI *bool `json:"enableCRI,omitempty"` // Enable dockershim only mode. // +optional ExperimentalDockershim *bool `json:"experimentalDockershim,omitempty"` diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index f4c1c3f1535..f607ea6174c 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -515,9 +515,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu } out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) out.FeatureGates = in.FeatureGates - if err := v1.Convert_Pointer_bool_To_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { - return err - } if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { return err } @@ -719,9 +716,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu } out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) out.FeatureGates = in.FeatureGates - if err := v1.Convert_bool_To_Pointer_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil { - return err - } if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil { return err } diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index 772699238c5..5770c9ac9ff 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -326,11 +326,6 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * *out = make([]string, len(*in)) copy(*out, *in) } - if in.EnableCRI != nil { - in, out := &in.EnableCRI, &out.EnableCRI - *out = new(bool) - **out = **in - } if in.ExperimentalDockershim != nil { in, out := &in.ExperimentalDockershim, &out.ExperimentalDockershim *out = new(bool) diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 86d64b85140..89d99e73c51 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -74,7 +74,6 @@ type NodeConfig struct { CgroupRoot string CgroupDriver string ProtectKernelDefaults bool - EnableCRI bool NodeAllocatableConfig ExperimentalQOSReserved map[v1.ResourceName]int64 } diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 6d855e37cec..2694be233a5 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -377,70 +377,25 @@ func (cm *containerManagerImpl) setupNode(activePods ActivePodsFunc) error { systemContainers := []*systemContainer{} if cm.ContainerRuntime == "docker" { - dockerAPIVersion := getDockerAPIVersion(cm.cadvisorInterface) - if cm.EnableCRI { - // If kubelet uses CRI, dockershim will manage the cgroups and oom - // score for the docker processes. - // In the future, NodeSpec should mandate the cgroup that the - // runtime processes need to be in. For now, we still check the - // cgroup for docker periodically, so that kubelet can recognize - // the cgroup for docker and serve stats for the runtime. - // TODO(#27097): Fix this after NodeSpec is clearly defined. - cm.periodicTasks = append(cm.periodicTasks, func() { - glog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration") - cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile) - if err != nil { - glog.Error(err) - return - } - glog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont) - cm.Lock() - defer cm.Unlock() - cm.RuntimeCgroupsName = cont - }) - } else if cm.RuntimeCgroupsName != "" { - cont := newSystemCgroups(cm.RuntimeCgroupsName) - memoryLimit := (int64(cm.capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100)) - if memoryLimit < MinDockerMemoryLimit { - glog.Warningf("Memory limit %d for container %s is too small, reset it to %d", memoryLimit, cm.RuntimeCgroupsName, MinDockerMemoryLimit) - memoryLimit = MinDockerMemoryLimit + // With the docker-CRI integration, dockershim will manage the cgroups + // and oom score for the docker processes. + // In the future, NodeSpec should mandate the cgroup that the + // runtime processes need to be in. For now, we still check the + // cgroup for docker periodically, so that kubelet can recognize + // the cgroup for docker and serve stats for the runtime. + // TODO(#27097): Fix this after NodeSpec is clearly defined. + cm.periodicTasks = append(cm.periodicTasks, func() { + glog.V(4).Infof("[ContainerManager]: Adding periodic tasks for docker CRI integration") + cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile) + if err != nil { + glog.Error(err) + return } - - glog.V(2).Infof("Configure resource-only container %s with memory limit: %d", cm.RuntimeCgroupsName, memoryLimit) - allowAllDevices := true - dockerContainer := &fs.Manager{ - Cgroups: &configs.Cgroup{ - Parent: "/", - Name: cm.RuntimeCgroupsName, - Resources: &configs.Resources{ - Memory: memoryLimit, - MemorySwap: -1, - AllowAllDevices: &allowAllDevices, - }, - }, - } - cont.ensureStateFunc = func(manager *fs.Manager) error { - return EnsureDockerInContainer(dockerAPIVersion, qos.DockerOOMScoreAdj, dockerContainer) - } - systemContainers = append(systemContainers, cont) - } else { - cm.periodicTasks = append(cm.periodicTasks, func() { - glog.V(10).Infof("Adding docker daemon periodic tasks") - if err := EnsureDockerInContainer(dockerAPIVersion, qos.DockerOOMScoreAdj, nil); err != nil { - glog.Error(err) - return - } - cont, err := getContainerNameForProcess(dockerProcessName, dockerPidFile) - if err != nil { - glog.Error(err) - return - } - glog.V(2).Infof("Discovered runtime cgroups name: %s", cont) - cm.Lock() - defer cm.Unlock() - cm.RuntimeCgroupsName = cont - }) - } + glog.V(2).Infof("[ContainerManager]: Discovered runtime cgroups name: %s", cont) + cm.Lock() + defer cm.Unlock() + cm.RuntimeCgroupsName = cont + }) } if cm.SystemCgroupsName != "" { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 047eda537d5..d2049d7c84d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -530,12 +530,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub pluginSettings.LegacyRuntimeHost = nl // rktnetes cannot be run with CRI. - // TODO(yujuhong): Remove the EnableCRI field. if kubeCfg.ContainerRuntime != "rkt" { - kubeCfg.EnableCRI = true - } - - if kubeCfg.EnableCRI { // kubelet defers to the runtime shim to setup networking. Setting // this to nil will prevent it from trying to invoke the plugin. // It's easier to always probe and initialize plugins till cri @@ -614,6 +609,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub klet.containerRuntime = runtime klet.runner = runtime } else { + // rkt uses the legacy, non-CRI, integration. Configure it the old way. // TODO: Include hairpin mode settings in rkt? conf := &rkt.Config{ Path: kubeCfg.RktPath, @@ -2018,9 +2014,9 @@ func (kl *Kubelet) updateRuntimeUp() { glog.Errorf("Container runtime sanity check failed: %v", err) return } - // Only check specific conditions when runtime integration type is cri, - // because the old integration doesn't populate any runtime condition. - if kl.kubeletConfiguration.EnableCRI && kl.kubeletConfiguration.ContainerRuntime != "rkt" { + // rkt uses the legacy, non-CRI integration. Don't check the runtime + // conditions for it. + if kl.kubeletConfiguration.ContainerRuntime != "rkt" { if s == nil { glog.Errorf("Container runtime status is nil") return diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index 7a4b021ae03..6d57dba0ce2 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -812,8 +812,6 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) { kubelet.updateRuntimeUp() checkNodeStatus(v1.ConditionFalse, "KubeletNotReady") - // Test cri integration. - kubelet.kubeletConfiguration.EnableCRI = true fakeRuntime.StatusErr = nil // Should report node not ready if runtime status is nil. diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 721565e4063..1aec70dde34 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -143,6 +143,14 @@ func newTestKubeletWithImageList( fakeRuntime.RuntimeType = "test" fakeRuntime.VersionInfo = "1.5.0" fakeRuntime.ImageList = imageList + // Set ready conditions by default. + fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{ + Conditions: []kubecontainer.RuntimeCondition{ + {Type: "RuntimeReady", Status: true}, + {Type: "NetworkReady", Status: true}, + }, + } + fakeRecorder := &record.FakeRecorder{} fakeKubeClient := &fake.Clientset{} kubelet := &Kubelet{}