Merge pull request #34877 from resouer/e2e-log-path

Automatic merge from submit-queue

Add e2e node test for log path

fixes #34661

A node e2e test to check if container logs files are properly created with right content.

Since the log files under `/var/log/containers` are actually symbolic of docker containers log files, we can not use a pod to mount them in and do check (symbolic doesn't supported by docker volume).

cc @Random-Liu

pkg/kubelet/kubelet.go

@@ -102,7 +102,7 @@ const (
 	nodeStatusUpdateRetry = 5
 
 	// Location of container logs.
-	containerLogsDir = "/var/log/containers"
+	ContainerLogsDir = "/var/log/containers"
 
 	// max backoff period, exported for the e2e test
 	MaxContainerBackOff = 300 * time.Second
@@ -617,7 +617,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 				kubeCfg.PodInfraContainerImage,
 				float32(kubeCfg.RegistryPullQPS),
 				int(kubeCfg.RegistryBurst),
-				containerLogsDir,
+				ContainerLogsDir,
 				kubeDeps.OSInterface,
 				klet.networkPlugin,
 				klet,
@@ -1156,9 +1156,9 @@ func (kl *Kubelet) initializeModules() error {
 	}
 
 	// Step 3: If the container logs directory does not exist, create it.
-	if _, err := os.Stat(containerLogsDir); err != nil {
-		if err := kl.os.MkdirAll(containerLogsDir, 0755); err != nil {
-			glog.Errorf("Failed to create directory %q: %v", containerLogsDir, err)
+	if _, err := os.Stat(ContainerLogsDir); err != nil {
+		if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
+			glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
 		}
 	}
 

pkg/kubelet/runonce.go

@@ -48,9 +48,9 @@ func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult,
 	}
 
 	// If the container logs directory does not exist, create it.
-	if _, err := os.Stat(containerLogsDir); err != nil {
-		if err := kl.os.MkdirAll(containerLogsDir, 0755); err != nil {
-			glog.Errorf("Failed to create directory %q: %v", containerLogsDir, err)
+	if _, err := os.Stat(ContainerLogsDir); err != nil {
+		if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
+			glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
 		}
 	}
 

test/e2e/generated/BUILD

@@ -13,28 +13,8 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "bindata.go",
         "main.go",
-        ":bindata",
-    ],
-)
-
-genrule(
-    name = "bindata",
-    srcs = [
-        "//examples:sources",
-        "//test/images:sources",
-        "//test/fixtures:sources",
-        "//test/e2e/testing-manifests:sources",
-    ],
-    outs = ["bindata.go"],
-    cmd = """
-$(location //vendor:github.com/jteeuwen/go-bindata/go-bindata) \
-  -nometadata -o "$(OUTS)" -pkg generated \
-  -prefix $$(pwd) \
-  -ignore .jpg -ignore .png -ignore .md \
-  $(SRCS)
-""",
-    tools = [
-        "//vendor:github.com/jteeuwen/go-bindata/go-bindata",
     ],
+    tags = ["automanaged"],
 )

test/e2e_node/BUILD

@@ -60,6 +60,7 @@ go_test(
         "image_id_test.go",
         "kubelet_test.go",
         "lifecycle_hook_test.go",
+        "log_path_test.go",
         "memory_eviction_test.go",
         "mirror_pod_test.go",
         "resource_usage_test.go",
@@ -76,8 +77,10 @@ go_test(
         "//pkg/api/unversioned:go_default_library",
         "//pkg/client/cache:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/kubelet:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/dockertools:go_default_library",
         "//pkg/kubelet/images:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",

test/e2e_node/log_path_test.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/kubelet"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/util/uuid"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const logString = "This is the expected log content of this node e2e test"
+
+var _ = framework.KubeDescribe("ContainerLogPath", func() {
+	f := framework.NewDefaultFramework("kubelet-container-log-path")
+	Describe("Pod with a container", func() {
+		Context("printed log to stdout", func() {
+			It("should print log to correct log path", func() {
+				podClient := f.PodClient()
+				ns := f.Namespace.Name
+
+				logDirVolumeName := "log-dir-vol"
+				logDir := kubelet.ContainerLogsDir
+
+				logPodName := "logger-" + string(uuid.NewUUID())
+				logContName := "logger-c-" + string(uuid.NewUUID())
+				checkPodName := "checker" + string(uuid.NewUUID())
+				checkContName := "checker-c-" + string(uuid.NewUUID())
+
+				logPod := &api.Pod{
+					ObjectMeta: api.ObjectMeta{
+						Name: logPodName,
+					},
+					Spec: api.PodSpec{
+						// this pod is expected to exit successfully
+						RestartPolicy: api.RestartPolicyNever,
+						Containers: []api.Container{
+							{
+								Image:   "gcr.io/google_containers/busybox:1.24",
+								Name:    logContName,
+								Command: []string{"sh", "-c", "echo " + logString},
+							},
+						},
+					},
+				}
+
+				podClient.Create(logPod)
+				err := framework.WaitForPodSuccessInNamespace(f.ClientSet, logPodName, ns)
+				framework.ExpectNoError(err, "Failed waiting for pod: %s to enter success state", logPodName)
+
+				// get containerID from created Pod
+				createdLogPod, err := podClient.Get(logPodName)
+				logConID := kubecontainer.ParseContainerID(createdLogPod.Status.ContainerStatuses[0].ContainerID)
+				framework.ExpectNoError(err, "Failed to get pod: %s", logPodName)
+
+				expectedlogFile := logDir + "/" + logPodName + "_" + ns + "_" + logContName + "-" + logConID.ID + ".log"
+
+				checkPod := &api.Pod{
+					ObjectMeta: api.ObjectMeta{
+						Name: checkPodName,
+					},
+					Spec: api.PodSpec{
+						// this pod is expected to exit successfully
+						RestartPolicy: api.RestartPolicyNever,
+						Containers: []api.Container{
+							{
+								Image: "gcr.io/google_containers/busybox:1.24",
+								Name:  checkContName,
+								// If we find expected log file and contains right content, exit 0
+								// else, keep checking until test timeout
+								Command: []string{"sh", "-c", "while true; do if [ -e " + expectedlogFile + " ] && grep -q " + logString + " " + expectedlogFile + "; then exit 0; fi; sleep 1; done"},
+								VolumeMounts: []api.VolumeMount{
+									{
+										Name: logDirVolumeName,
+										// mount ContainerLogsDir to the same path in container
+										MountPath: expectedlogFile,
+										ReadOnly:  true,
+									},
+								},
+							},
+						},
+						Volumes: []api.Volume{
+							{
+								Name: logDirVolumeName,
+								VolumeSource: api.VolumeSource{
+									HostPath: &api.HostPathVolumeSource{
+										Path: expectedlogFile,
+									},
+								},
+							},
+						},
+					},
+				}
+
+				podClient.Create(checkPod)
+				err = framework.WaitForPodSuccessInNamespace(f.ClientSet, checkPodName, ns)
+				framework.ExpectNoError(err, "Failed waiting for pod: %s to enter success state", checkPodName)
+			})
+		})
+	})
+})

test/test_owners.csv

@@ -45,6 +45,7 @@ CronJob should replace jobs when ReplaceConcurrent,soltysh,1
 CronJob should schedule multiple jobs concurrently,soltysh,1
 DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios,MrHohn,0
 DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods when cluster size changed,MrHohn,0
+ContainerLogPath Pod with a container printed log to stdout should print log to correct log path,resouer,0
 DNS should provide DNS for ExternalName services,rmmh,1
 DNS should provide DNS for pods for Hostname and Subdomain Annotation,mtaufen,1
 DNS should provide DNS for services,roberthbailey,1