diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 2f5f4e3d188..28d58b11108 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -265,6 +265,13 @@ const ( ContainerStateUnknown ContainerState = "unknown" ) +type ContainerType string + +const ( + ContainerTypeInit ContainerType = "INIT" + ContainerTypeRegular ContainerType = "REGULAR" +) + // Container provides the runtime information for a container, such as ID, hash, // state of the container. type Container struct { diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 9438ec410d3..c27f04ef318 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -29,6 +29,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/credentialprovider:go_default_library", "//pkg/credentialprovider/secrets:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/kubelet/cm:go_default_library", @@ -59,6 +60,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/tools/reference:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", @@ -99,6 +101,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", ], ) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index de0b6fdd7f3..5f81a0d8f39 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -85,7 +85,7 @@ func (m *kubeGenericRuntimeManager) recordContainerEvent(pod *v1.Pod, container // * create the container // * start the container // * run the post start lifecycle hooks (if applicable) -func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, container *v1.Container, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string) (string, error) { +func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, container *v1.Container, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, containerType kubecontainer.ContainerType) (string, error) { // Step 1: pull the image. imageRef, msg, err := m.imagePuller.EnsureImageExists(pod, container, pullSecrets) if err != nil { @@ -107,7 +107,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb restartCount = containerStatus.RestartCount + 1 } - containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef) + containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType) if err != nil { m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err)) return grpc.ErrorDesc(err), ErrCreateContainerConfig @@ -173,7 +173,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb } // generateContainerConfig generates container config for kubelet runtime v1. -func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string) (*runtimeapi.ContainerConfig, error) { +func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, error) { opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) if err != nil { return nil, err @@ -201,7 +201,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai Command: command, Args: args, WorkingDir: container.WorkingDir, - Labels: newContainerLabels(container, pod), + Labels: newContainerLabels(container, pod, containerType), Annotations: newContainerAnnotations(container, pod, restartCount), Devices: makeDevices(opts), Mounts: m.makeMounts(opts, container), diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index c91a3a82e3f..45f5e9809c9 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -221,7 +221,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde Command: container.Command, Args: []string(nil), WorkingDir: container.WorkingDir, - Labels: newContainerLabels(container, pod), + Labels: newContainerLabels(container, pod, kubecontainer.ContainerTypeRegular), Annotations: newContainerAnnotations(container, pod, restartCount), Devices: makeDevices(opts), Mounts: m.makeMounts(opts, container), @@ -259,7 +259,7 @@ func TestGenerateContainerConfig(t *testing.T) { } expectedConfig := makeExpectedConfig(m, pod, 0) - containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image) + containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular) assert.NoError(t, err) assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.") @@ -288,7 +288,7 @@ func TestGenerateContainerConfig(t *testing.T) { }, } - _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular) assert.Error(t, err) imageId, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil) @@ -300,7 +300,7 @@ func TestGenerateContainerConfig(t *testing.T) { podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue - _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular) assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username") } @@ -424,7 +424,7 @@ func TestLifeCycleHook(t *testing.T) { } // Now try to create a container, which should in turn invoke PostStart Hook - _, err := m.startContainer(fakeSandBox.Id, fakeSandBoxConfig, testContainer, testPod, fakePodStatus, nil, "") + _, err := m.startContainer(fakeSandBox.Id, fakeSandBoxConfig, testContainer, testPod, fakePodStatus, nil, "", kubecontainer.ContainerTypeRegular) if err != nil { t.Errorf("startContainer erro =%v", err) } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index c63e02dc8bc..3b7a9e66a36 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -699,7 +699,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat } glog.V(4).Infof("Creating init container %+v in pod %v", container, format.Pod(pod)) - if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP); err != nil { + if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeInit); err != nil { startContainerResult.Fail(err, msg) utilruntime.HandleError(fmt.Errorf("init container start failed: %v: %s", err, msg)) return @@ -723,7 +723,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat } glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod)) - if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP); err != nil { + if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeRegular); err != nil { startContainerResult.Fail(err, msg) // known errors that are logged in other places are logged at higher levels here to avoid // repetitive log spam diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index f2ae5281117..c688f077da9 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -70,6 +70,7 @@ type sandboxTemplate struct { type containerTemplate struct { pod *v1.Pod container *v1.Container + containerType kubecontainer.ContainerType sandboxAttempt uint32 attempt int createdAt int64 @@ -142,7 +143,7 @@ func makeFakeContainer(t *testing.T, m *kubeGenericRuntimeManager, template cont sandboxConfig, err := m.generatePodSandboxConfig(template.pod, template.sandboxAttempt) assert.NoError(t, err, "generatePodSandboxConfig for container template %+v", template) - containerConfig, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image) + containerConfig, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType) assert.NoError(t, err, "generateContainerConfig for container template %+v", template) podSandboxID := apitest.BuildSandboxName(sandboxConfig.Metadata) diff --git a/pkg/kubelet/kuberuntime/labels.go b/pkg/kubelet/kuberuntime/labels.go index d3da2cf8eb9..041c9239736 100644 --- a/pkg/kubelet/kuberuntime/labels.go +++ b/pkg/kubelet/kuberuntime/labels.go @@ -23,6 +23,8 @@ import ( "github.com/golang/glog" "k8s.io/api/core/v1" kubetypes "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -56,6 +58,7 @@ type annotatedPodSandboxInfo struct { type labeledContainerInfo struct { ContainerName string + ContainerType kubecontainer.ContainerType PodName string PodNamespace string PodUID kubetypes.UID @@ -94,12 +97,15 @@ func newPodAnnotations(pod *v1.Pod) map[string]string { } // newContainerLabels creates container labels from v1.Container and v1.Pod. -func newContainerLabels(container *v1.Container, pod *v1.Pod) map[string]string { +func newContainerLabels(container *v1.Container, pod *v1.Pod, containerType kubecontainer.ContainerType) map[string]string { labels := map[string]string{} labels[types.KubernetesPodNameLabel] = pod.Name labels[types.KubernetesPodNamespaceLabel] = pod.Namespace labels[types.KubernetesPodUIDLabel] = string(pod.UID) labels[types.KubernetesContainerNameLabel] = container.Name + if utilfeature.DefaultFeatureGate.Enabled(features.DebugContainers) { + labels[types.KubernetesContainerTypeLabel] = string(containerType) + } return labels } @@ -169,11 +175,16 @@ func getPodSandboxInfoFromAnnotations(annotations map[string]string) *annotatedP // getContainerInfoFromLabels gets labeledContainerInfo from labels. func getContainerInfoFromLabels(labels map[string]string) *labeledContainerInfo { + var containerType kubecontainer.ContainerType + if utilfeature.DefaultFeatureGate.Enabled(features.DebugContainers) { + containerType = kubecontainer.ContainerType(getStringValueFromLabel(labels, types.KubernetesContainerTypeLabel)) + } return &labeledContainerInfo{ PodName: getStringValueFromLabel(labels, types.KubernetesPodNameLabel), PodNamespace: getStringValueFromLabel(labels, types.KubernetesPodNamespaceLabel), PodUID: kubetypes.UID(getStringValueFromLabel(labels, types.KubernetesPodUIDLabel)), ContainerName: getStringValueFromLabel(labels, types.KubernetesContainerNameLabel), + ContainerType: containerType, } } diff --git a/pkg/kubelet/kuberuntime/labels_test.go b/pkg/kubelet/kuberuntime/labels_test.go index 941c9f66243..a8a3bbd2fe7 100644 --- a/pkg/kubelet/kuberuntime/labels_test.go +++ b/pkg/kubelet/kuberuntime/labels_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -63,19 +64,92 @@ func TestContainerLabels(t *testing.T) { TerminationGracePeriodSeconds: &terminationGracePeriod, }, } - expected := &labeledContainerInfo{ - PodName: pod.Name, - PodNamespace: pod.Namespace, - PodUID: pod.UID, - ContainerName: container.Name, + + var tests = []struct { + description string + featuresCreated string // Features enabled when container is created + featuresStatus string // Features enabled when container status is read + typeLabel kubecontainer.ContainerType + expected *labeledContainerInfo + }{ + { + "Debug containers disabled", + "DebugContainers=False", + "DebugContainers=False", + "ignored", + &labeledContainerInfo{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: pod.UID, + ContainerName: container.Name, + ContainerType: "", + }, + }, + { + "Regular containers", + "DebugContainers=True", + "DebugContainers=True", + kubecontainer.ContainerTypeRegular, + &labeledContainerInfo{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: pod.UID, + ContainerName: container.Name, + ContainerType: kubecontainer.ContainerTypeRegular, + }, + }, + { + "Init containers", + "DebugContainers=True", + "DebugContainers=True", + kubecontainer.ContainerTypeInit, + &labeledContainerInfo{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: pod.UID, + ContainerName: container.Name, + ContainerType: kubecontainer.ContainerTypeInit, + }, + }, + { + "Created without type label", + "DebugContainers=False", + "DebugContainers=True", + "ignored", + &labeledContainerInfo{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: pod.UID, + ContainerName: container.Name, + ContainerType: "", + }, + }, + { + "Created with type label, subsequently disabled", + "DebugContainers=True", + "DebugContainers=False", + kubecontainer.ContainerTypeRegular, + &labeledContainerInfo{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: pod.UID, + ContainerName: container.Name, + ContainerType: "", + }, + }, } // Test whether we can get right information from label - labels := newContainerLabels(container, pod) - containerInfo := getContainerInfoFromLabels(labels) - if !reflect.DeepEqual(containerInfo, expected) { - t.Errorf("expected %v, got %v", expected, containerInfo) + for _, test := range tests { + utilfeature.DefaultFeatureGate.Set(test.featuresCreated) + labels := newContainerLabels(container, pod, test.typeLabel) + utilfeature.DefaultFeatureGate.Set(test.featuresStatus) + containerInfo := getContainerInfoFromLabels(labels) + if !reflect.DeepEqual(containerInfo, test.expected) { + t.Errorf("%v: expected %v, got %v", test.description, test.expected, containerInfo) + } } + utilfeature.DefaultFeatureGate.Set("DebugContainers=False") } func TestContainerAnnotations(t *testing.T) { diff --git a/pkg/kubelet/types/labels.go b/pkg/kubelet/types/labels.go index c4dad6302e5..67c84f6d614 100644 --- a/pkg/kubelet/types/labels.go +++ b/pkg/kubelet/types/labels.go @@ -21,6 +21,7 @@ const ( KubernetesPodNamespaceLabel = "io.kubernetes.pod.namespace" KubernetesPodUIDLabel = "io.kubernetes.pod.uid" KubernetesContainerNameLabel = "io.kubernetes.container.name" + KubernetesContainerTypeLabel = "io.kubernetes.container.type" ) func GetContainerName(labels map[string]string) string {