diff --git a/test/e2e/common/node/configmap.go b/test/e2e/common/node/configmap.go index 0c3d0811c3f..aa894afac83 100644 --- a/test/e2e/common/node/configmap.go +++ b/test/e2e/common/node/configmap.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -79,7 +80,7 @@ var _ = SIGDescribe("ConfigMap", func() { }, } - f.TestContainerOutput("consume configMaps", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{ "CONFIG_DATA_1=value-1", }) }) @@ -123,7 +124,7 @@ var _ = SIGDescribe("ConfigMap", func() { }, } - f.TestContainerOutput("consume configMaps", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{ "data-1=value-1", "data-2=value-2", "data-3=value-3", "p-data-1=value-1", "p-data-2=value-2", "p-data-3=value-3", }) diff --git a/test/e2e/common/node/containers.go b/test/e2e/common/node/containers.go index 5c1ac026e1b..47f45170642 100644 --- a/test/e2e/common/node/containers.go +++ b/test/e2e/common/node/containers.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" admissionapi "k8s.io/pod-security-admission/api" ) @@ -57,7 +58,7 @@ var _ = SIGDescribe("Containers", func() { */ framework.ConformanceIt("should be able to override the image's default arguments (container cmd) [NodeConformance]", func() { pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester", "override", "arguments") - f.TestContainerOutput("override arguments", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "override arguments", pod, 0, []string{ "[/agnhost entrypoint-tester override arguments]", }) }) @@ -73,7 +74,7 @@ var _ = SIGDescribe("Containers", func() { pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester") pod.Spec.Containers[0].Command = []string{"/agnhost-2"} - f.TestContainerOutput("override command", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "override command", pod, 0, []string{ "[/agnhost-2 entrypoint-tester]", }) }) @@ -87,7 +88,7 @@ var _ = SIGDescribe("Containers", func() { pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester", "override", "arguments") pod.Spec.Containers[0].Command = []string{"/agnhost-2"} - f.TestContainerOutput("override all", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "override all", pod, 0, []string{ "[/agnhost-2 entrypoint-tester override arguments]", }) }) diff --git a/test/e2e/common/node/downwardapi.go b/test/e2e/common/node/downwardapi.go index 2addc5366ee..8145794b9ba 100644 --- a/test/e2e/common/node/downwardapi.go +++ b/test/e2e/common/node/downwardapi.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -418,5 +419,5 @@ func testDownwardAPI(f *framework.Framework, podName string, env []v1.EnvVar, ex } func testDownwardAPIUsingPod(f *framework.Framework, pod *v1.Pod, env []v1.EnvVar, expectations []string) { - f.TestContainerOutputRegexp("downward api env vars", pod, 0, expectations) + e2etodopod.TestContainerOutputRegexp(f, "downward api env vars", pod, 0, expectations) } diff --git a/test/e2e/common/node/expansion.go b/test/e2e/common/node/expansion.go index 62c86dde067..a8890ff5b75 100644 --- a/test/e2e/common/node/expansion.go +++ b/test/e2e/common/node/expansion.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -57,7 +58,7 @@ var _ = SIGDescribe("Variable Expansion", func() { } pod := newPod([]string{"sh", "-c", "env"}, envVars, nil, nil) - f.TestContainerOutput("env composition", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "env composition", pod, 0, []string{ "FOO=foo-value", "BAR=bar-value", "FOOBAR=foo-value;;bar-value", @@ -78,7 +79,7 @@ var _ = SIGDescribe("Variable Expansion", func() { } pod := newPod([]string{"sh", "-c", "TEST_VAR=wrong echo \"$(TEST_VAR)\""}, envVars, nil, nil) - f.TestContainerOutput("substitution in container's command", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "substitution in container's command", pod, 0, []string{ "test-value", }) }) @@ -98,7 +99,7 @@ var _ = SIGDescribe("Variable Expansion", func() { pod := newPod([]string{"sh", "-c"}, envVars, nil, nil) pod.Spec.Containers[0].Args = []string{"TEST_VAR=wrong echo \"$(TEST_VAR)\""} - f.TestContainerOutput("substitution in container's args", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "substitution in container's args", pod, 0, []string{ "test-value", }) }) @@ -138,7 +139,7 @@ var _ = SIGDescribe("Variable Expansion", func() { envVars[0].Value = pod.ObjectMeta.Name pod.Spec.Containers[0].Command = []string{"sh", "-c", "test -d /testcontainer/" + pod.ObjectMeta.Name + ";echo $?"} - f.TestContainerOutput("substitution in volume subpath", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "substitution in volume subpath", pod, 0, []string{ "0", }) }) diff --git a/test/e2e/common/node/pods.go b/test/e2e/common/node/pods.go index 87adee092ae..be6f6d52ae2 100644 --- a/test/e2e/common/node/pods.go +++ b/test/e2e/common/node/pods.go @@ -51,6 +51,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" e2ewebsocket "k8s.io/kubernetes/test/e2e/framework/websocket" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -522,7 +523,7 @@ var _ = SIGDescribe("Pods", func() { "FOOSERVICE_PORT_8765_TCP_ADDR=", } expectNoErrorWithRetries(func() error { - return f.MatchContainerOutput(pod, containerName, expectedVars, gomega.ContainSubstring) + return e2etodopod.MatchContainerOutput(f, pod, containerName, expectedVars, gomega.ContainSubstring) }, maxRetries, "Container should have service environment variables set") }) diff --git a/test/e2e/common/node/secrets.go b/test/e2e/common/node/secrets.go index 7ff0130a70f..2738f5d900f 100644 --- a/test/e2e/common/node/secrets.go +++ b/test/e2e/common/node/secrets.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -81,7 +82,7 @@ var _ = SIGDescribe("Secrets", func() { }, } - f.TestContainerOutput("consume secrets", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume secrets", pod, 0, []string{ "SECRET_DATA=value-1", }) }) @@ -125,7 +126,7 @@ var _ = SIGDescribe("Secrets", func() { }, } - f.TestContainerOutput("consume secrets", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume secrets", pod, 0, []string{ "data-1=value-1", "data-2=value-2", "data-3=value-3", "p-data-1=value-1", "p-data-2=value-2", "p-data-3=value-3", }) diff --git a/test/e2e/common/node/security_context.go b/test/e2e/common/node/security_context.go index acfe36fc287..d914af8039c 100644 --- a/test/e2e/common/node/security_context.go +++ b/test/e2e/common/node/security_context.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/pointer" @@ -115,7 +116,7 @@ var _ = SIGDescribe("Security Context", func() { // When running in the host's user namespace, the /proc/self/uid_map file content looks like: // 0 0 4294967295 // Verify the value 4294967295 is present in the output. - f.TestContainerOutput("read namespace", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "read namespace", pod, 0, []string{ "4294967295", }) }) @@ -239,7 +240,7 @@ var _ = SIGDescribe("Security Context", func() { // Each line should be "=0" that means root inside the container is the owner of the file. downwardAPIVolFiles := 1 projectedFiles := len(secret.Data) + downwardAPIVolFiles - f.TestContainerOutput("check file permissions", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "check file permissions", pod, 0, []string{ strings.Repeat("=0\n", len(secret.Data)+len(configMap.Data)+downwardAPIVolFiles+projectedFiles), }) }) @@ -299,7 +300,7 @@ var _ = SIGDescribe("Security Context", func() { // Expect one line for each file on all the volumes. // Each line should be "=200" (fsGroup) that means it was mapped to the // right user inside the container. - f.TestContainerOutput("check FSGroup is mapped correctly", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "check FSGroup is mapped correctly", pod, 0, []string{ strings.Repeat(fmt.Sprintf("=%v\n", fsGroup), len(configMap.Data)), }) }) diff --git a/test/e2e/common/storage/configmap_volume.go b/test/e2e/common/storage/configmap_volume.go index 9c60cf7cc40..c2d4033b10a 100644 --- a/test/e2e/common/storage/configmap_volume.go +++ b/test/e2e/common/storage/configmap_volume.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -485,7 +486,7 @@ var _ = SIGDescribe("ConfigMap", func() { }, } - f.TestContainerOutput("consume configMaps", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{ "content of file \"/etc/configmap-volume/data-1\": value-1", }) @@ -621,7 +622,7 @@ func doConfigMapE2EWithoutMappings(f *framework.Framework, asUser bool, fsGroup "content of file \"/etc/configmap-volume/data-1\": value-1", fileModeRegexp, } - f.TestContainerOutputRegexp("consume configMaps", pod, 0, output) + e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output) } func doConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int64, itemMode *int32) { @@ -673,7 +674,7 @@ func doConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int fileModeRegexp := getFileModeRegex("/etc/configmap-volume/path/to/data-2", itemMode) output = append(output, fileModeRegexp) } - f.TestContainerOutputRegexp("consume configMaps", pod, 0, output) + e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output) } func createNonOptionalConfigMapPod(f *framework.Framework, volumeMountPath string) (*v1.Pod, error) { diff --git a/test/e2e/common/storage/downwardapi.go b/test/e2e/common/storage/downwardapi.go index eec583f4bf9..9dc385ff4fc 100644 --- a/test/e2e/common/storage/downwardapi.go +++ b/test/e2e/common/storage/downwardapi.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -133,5 +134,5 @@ func testDownwardAPIForEphemeralStorage(f *framework.Framework, podName string, } func testDownwardAPIUsingPod(f *framework.Framework, pod *v1.Pod, env []v1.EnvVar, expectations []string) { - f.TestContainerOutputRegexp("downward api env vars", pod, 0, expectations) + e2etodopod.TestContainerOutputRegexp(f, "downward api env vars", pod, 0, expectations) } diff --git a/test/e2e/common/storage/downwardapi_volume.go b/test/e2e/common/storage/downwardapi_volume.go index b7bb6a153d4..b6f5efb48c7 100644 --- a/test/e2e/common/storage/downwardapi_volume.go +++ b/test/e2e/common/storage/downwardapi_volume.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -53,7 +54,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), }) }) @@ -69,7 +70,7 @@ var _ = SIGDescribe("Downward API volume", func() { defaultMode := int32(0400) pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -85,7 +86,7 @@ var _ = SIGDescribe("Downward API volume", func() { mode := int32(0400) pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -100,7 +101,7 @@ var _ = SIGDescribe("Downward API volume", func() { FSGroup: &gid, } setPodNonRootUser(pod) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), }) }) @@ -116,7 +117,7 @@ var _ = SIGDescribe("Downward API volume", func() { FSGroup: &gid, } setPodNonRootUser(pod) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--r-----", }) }) @@ -193,7 +194,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("2\n"), }) }) @@ -207,7 +208,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("67108864\n"), }) }) @@ -221,7 +222,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("1\n"), }) }) @@ -235,7 +236,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("33554432\n"), }) }) @@ -249,7 +250,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit") - f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) + e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"}) }) /* @@ -261,7 +262,7 @@ var _ = SIGDescribe("Downward API volume", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit") - f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) + e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"}) }) }) diff --git a/test/e2e/common/storage/empty_dir.go b/test/e2e/common/storage/empty_dir.go index 8c676c206bd..3087b80e267 100644 --- a/test/e2e/common/storage/empty_dir.go +++ b/test/e2e/common/storage/empty_dir.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -390,7 +391,7 @@ func doTestSetgidFSGroup(f *framework.Framework, uid int64, medium v1.StorageMed if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTestSubPathFSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -423,7 +424,7 @@ func doTestSubPathFSGroup(f *framework.Framework, uid int64, medium v1.StorageMe if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTestVolumeModeFSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -448,7 +449,7 @@ func doTestVolumeModeFSGroup(f *framework.Framework, uid int64, medium v1.Storag if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTest0644FSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -476,7 +477,7 @@ func doTest0644FSGroup(f *framework.Framework, uid int64, medium v1.StorageMediu if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTestVolumeMode(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -498,7 +499,7 @@ func doTestVolumeMode(f *framework.Framework, uid int64, medium v1.StorageMedium if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTest0644(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -523,7 +524,7 @@ func doTest0644(f *framework.Framework, uid int64, medium v1.StorageMedium) { if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTest0666(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -548,7 +549,7 @@ func doTest0666(f *framework.Framework, uid int64, medium v1.StorageMedium) { if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func doTest0777(f *framework.Framework, uid int64, medium v1.StorageMedium) { @@ -573,7 +574,7 @@ func doTest0777(f *framework.Framework, uid int64, medium v1.StorageMedium) { if medium == v1.StorageMediumMemory { out = append(out, "mount type of \"/test-volume\": tmpfs") } - f.TestContainerOutput(msg, pod, 0, out) + e2etodopod.TestContainerOutput(f, msg, pod, 0, out) } func formatMedium(medium v1.StorageMedium) string { diff --git a/test/e2e/common/storage/host_path.go b/test/e2e/common/storage/host_path.go index 1d3526fddec..e606ce0e05c 100644 --- a/test/e2e/common/storage/host_path.go +++ b/test/e2e/common/storage/host_path.go @@ -21,9 +21,10 @@ import ( "os" "path" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -58,7 +59,7 @@ var _ = SIGDescribe("HostPath", func() { fmt.Sprintf("--fs_type=%v", volumePath), fmt.Sprintf("--file_mode=%v", volumePath), } - f.TestContainerOutputRegexp("hostPath mode", pod, 0, []string{ + e2etodopod.TestContainerOutputRegexp(f, "hostPath mode", pod, 0, []string{ "mode of file \"/test-volume\": dg?trwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir }) }) @@ -87,7 +88,7 @@ var _ = SIGDescribe("HostPath", func() { } //Read the content of the file with the second container to //verify volumes being shared properly among containers within the pod. - f.TestContainerOutput("hostPath r/w", pod, 1, []string{ + e2etodopod.TestContainerOutput(f, "hostPath r/w", pod, 1, []string{ "content of file \"/test-volume/test-file\": mount-tester new file", }) }) @@ -124,7 +125,7 @@ var _ = SIGDescribe("HostPath", func() { fmt.Sprintf("--retry_time=%d", retryDuration), } - f.TestContainerOutput("hostPath subPath", pod, 1, []string{ + e2etodopod.TestContainerOutput(f, "hostPath subPath", pod, 1, []string{ "content of file \"" + filePathInReader + "\": mount-tester new file", }) }) diff --git a/test/e2e/common/storage/projected_combined.go b/test/e2e/common/storage/projected_combined.go index 0ed2d8a0bf5..54fed314da5 100644 --- a/test/e2e/common/storage/projected_combined.go +++ b/test/e2e/common/storage/projected_combined.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -88,7 +89,7 @@ var _ = SIGDescribe("Projected combined", func() { }, }, } - f.TestContainerOutput("Check all projections for projected volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "Check all projections for projected volume plugin", pod, 0, []string{ podName, "secret-value-1", "configmap-value-1", diff --git a/test/e2e/common/storage/projected_configmap.go b/test/e2e/common/storage/projected_configmap.go index 9fc348babaa..d7b67df7c12 100644 --- a/test/e2e/common/storage/projected_configmap.go +++ b/test/e2e/common/storage/projected_configmap.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -450,7 +451,7 @@ var _ = SIGDescribe("Projected configMap", func() { }, } - f.TestContainerOutput("consume configMaps", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{ "content of file \"/etc/projected-configmap-volume/data-1\": value-1", }) @@ -512,7 +513,7 @@ func doProjectedConfigMapE2EWithoutMappings(f *framework.Framework, asUser bool, "content of file \"/etc/projected-configmap-volume/data-1\": value-1", fileModeRegexp, } - f.TestContainerOutputRegexp("consume configMaps", pod, 0, output) + e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output) } func doProjectedConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int64, itemMode *int32) { @@ -563,7 +564,7 @@ func doProjectedConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fs fileModeRegexp := getFileModeRegex("/etc/projected-configmap-volume/path/to/data-2", itemMode) output = append(output, fileModeRegexp) } - f.TestContainerOutputRegexp("consume configMaps", pod, 0, output) + e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output) } func createProjectedConfigMapMounttestPod(namespace, volumeName, referenceName, mountPath string, mounttestArgs ...string) *v1.Pod { diff --git a/test/e2e/common/storage/projected_downwardapi.go b/test/e2e/common/storage/projected_downwardapi.go index 78f93474166..65374c7e110 100644 --- a/test/e2e/common/storage/projected_downwardapi.go +++ b/test/e2e/common/storage/projected_downwardapi.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -53,7 +54,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), }) }) @@ -69,7 +70,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { defaultMode := int32(0400) pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -85,7 +86,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { mode := int32(0400) pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -100,7 +101,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { FSGroup: &gid, } setPodNonRootUser(pod) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), }) }) @@ -116,7 +117,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { FSGroup: &gid, } setPodNonRootUser(pod) - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ "mode of file \"/etc/podinfo/podname\": -r--r-----", }) }) @@ -193,7 +194,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("2\n"), }) }) @@ -207,7 +208,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("67108864\n"), }) }) @@ -221,7 +222,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("1\n"), }) }) @@ -235,7 +236,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request") - f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ + e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{ fmt.Sprintf("33554432\n"), }) }) @@ -249,7 +250,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit") - f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) + e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"}) }) /* @@ -261,7 +262,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit") - f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) + e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"}) }) }) diff --git a/test/e2e/common/storage/projected_secret.go b/test/e2e/common/storage/projected_secret.go index b16360db80f..f3d65d032ef 100644 --- a/test/e2e/common/storage/projected_secret.go +++ b/test/e2e/common/storage/projected_secret.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -200,7 +201,7 @@ var _ = SIGDescribe("Projected secret", func() { } fileModeRegexp := getFileModeRegex("/etc/projected-secret-volume/data-1", nil) - f.TestContainerOutputRegexp("consume secrets", pod, 0, []string{ + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, []string{ "content of file \"/etc/projected-secret-volume/data-1\": value-1", fileModeRegexp, }) @@ -504,7 +505,7 @@ func doProjectedSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int fileModeRegexp, } - f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput) + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput) } func doProjectedSecretE2EWithMapping(f *framework.Framework, mode *int32) { @@ -581,5 +582,5 @@ func doProjectedSecretE2EWithMapping(f *framework.Framework, mode *int32) { fileModeRegexp, } - f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput) + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput) } diff --git a/test/e2e/common/storage/secrets_volume.go b/test/e2e/common/storage/secrets_volume.go index 2eb7bf408fc..9dedcf6bace 100644 --- a/test/e2e/common/storage/secrets_volume.go +++ b/test/e2e/common/storage/secrets_volume.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -190,7 +191,7 @@ var _ = SIGDescribe("Secrets", func() { } fileModeRegexp := getFileModeRegex("/etc/secret-volume/data-1", nil) - f.TestContainerOutputRegexp("consume secrets", pod, 0, []string{ + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, []string{ "content of file \"/etc/secret-volume/data-1\": value-1", fileModeRegexp, }) @@ -534,7 +535,7 @@ func doSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int32, secre fileModeRegexp, } - f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput) + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput) } func doSecretE2EWithMapping(f *framework.Framework, mode *int32) { @@ -602,7 +603,7 @@ func doSecretE2EWithMapping(f *framework.Framework, mode *int32) { fileModeRegexp, } - f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput) + e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput) } func createNonOptionalSecretPod(f *framework.Framework, volumeMountPath, podName string) error { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index a00765e5ff5..072f81633dc 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -50,7 +50,6 @@ import ( admissionapi "k8s.io/pod-security-admission/api" "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" // TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245) e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" @@ -563,20 +562,6 @@ func (f *Framework) ClientConfig() *rest.Config { return ret } -// TestContainerOutput runs the given pod in the given namespace and waits -// for all of the containers in the podSpec to move into the 'Success' status, and tests -// the specified container log against the given expected output using a substring matcher. -func (f *Framework) TestContainerOutput(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) { - f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, gomega.ContainSubstring) -} - -// TestContainerOutputRegexp runs the given pod in the given namespace and waits -// for all of the containers in the podSpec to move into the 'Success' status, and tests -// the specified container log against the given expected output using a regexp matcher. -func (f *Framework) TestContainerOutputRegexp(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) { - f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, gomega.MatchRegexp) -} - // KubeUser is a struct for managing kubernetes user info. type KubeUser struct { Name string `yaml:"name"` diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index f1a01b48ba3..3f47f0a1f92 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -61,6 +61,7 @@ import ( e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" + e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -465,10 +466,10 @@ func (j *TestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[stri } j.Logger.Infof("creating replication controller") - framework.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-") + e2etodokubectl.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-") j.Logger.Infof("creating service") - framework.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-") + e2etodokubectl.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-") if len(svcAnnotations) > 0 { svcList, err := j.Client.CoreV1().Services(ns).List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err) @@ -481,7 +482,7 @@ func (j *TestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[stri if exists("secret.yaml") { j.Logger.Infof("creating secret") - framework.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-") + e2etodokubectl.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-") } j.Logger.Infof("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml")) @@ -550,7 +551,7 @@ func (j *TestJig) runCreate(ing *networkingv1.Ingress) (*networkingv1.Ingress, e if err := ingressToManifest(ing, filePath); err != nil { return nil, err } - _, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) + _, err := e2etodokubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) return ing, err } @@ -565,14 +566,14 @@ func (j *TestJig) runUpdate(ing *networkingv1.Ingress) (*networkingv1.Ingress, e if err := ingressToManifest(ing, filePath); err != nil { return nil, err } - _, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force") + _, err := e2etodokubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force") return ing, err } // DescribeIng describes information of ingress by running kubectl describe ing. func DescribeIng(ns string) { framework.Logf("\nOutput of kubectl describe ing:\n") - desc, _ := framework.RunKubectl( + desc, _ := e2etodokubectl.RunKubectl( ns, "describe", "ing") framework.Logf(desc) } @@ -680,7 +681,7 @@ func (j *TestJig) runDelete(ing *networkingv1.Ingress) error { if err := ingressToManifest(ing, filePath); err != nil { return err } - _, err := framework.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) + _, err := e2etodokubectl.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) return err } @@ -688,7 +689,7 @@ func (j *TestJig) runDelete(ing *networkingv1.Ingress) error { // TODO(nikhiljindal): Update this to be able to return hostname as well. func getIngressAddressFromKubemci(name string) ([]string, error) { var addresses []string - out, err := framework.RunKubemciCmd("get-status", name) + out, err := e2etodokubectl.RunKubemciCmd("get-status", name) if err != nil { return addresses, err } @@ -1032,7 +1033,7 @@ func (cont *NginxIngressController) Init() { } framework.Logf("initializing nginx ingress controller") - framework.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-") + e2etodokubectl.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-") rc, err := cont.Client.CoreV1().ReplicationControllers(cont.Ns).Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) framework.ExpectNoError(err) diff --git a/test/e2e/framework/network/utils.go b/test/e2e/framework/network/utils.go index 3dfd28435b7..d6277dc4c55 100644 --- a/test/e2e/framework/network/utils.go +++ b/test/e2e/framework/network/utils.go @@ -45,6 +45,8 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" + e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" netutils "k8s.io/utils/net" ) @@ -248,7 +250,7 @@ func (config *NetworkingTestConfig) diagnoseMissingEndpoints(foundEndpoints sets continue } framework.Logf("\nOutput of kubectl describe pod %v/%v:\n", e.Namespace, e.Name) - desc, _ := framework.RunKubectl( + desc, _ := e2etodokubectl.RunKubectl( e.Namespace, "describe", "pod", e.Name, fmt.Sprintf("--namespace=%v", e.Namespace)) framework.Logf(desc) } @@ -519,7 +521,7 @@ func (config *NetworkingTestConfig) executeCurlCmd(cmd string, expected string) podName := config.HostTestContainerPod.Name var msg string if pollErr := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - stdout, err := framework.RunHostCmd(config.Namespace, podName, cmd) + stdout, err := e2etodopod.RunHostCmd(config.Namespace, podName, cmd) if err != nil { msg = fmt.Sprintf("failed executing cmd %v in %v/%v: %v", cmd, config.Namespace, podName, err) framework.Logf(msg) @@ -533,7 +535,7 @@ func (config *NetworkingTestConfig) executeCurlCmd(cmd string, expected string) return true, nil }); pollErr != nil { framework.Logf("\nOutput of kubectl describe pod %v/%v:\n", config.Namespace, podName) - desc, _ := framework.RunKubectl( + desc, _ := e2etodokubectl.RunKubectl( config.Namespace, "describe", "pod", podName, fmt.Sprintf("--namespace=%v", config.Namespace)) framework.Logf("%s", desc) framework.Failf("Timed out in %v: %v", retryTimeout, msg) diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index c97b1fe1543..f3231e7ed2c 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -17,9 +17,6 @@ limitations under the License. package framework import ( - "fmt" - "os" - "path" "sync" "time" @@ -34,38 +31,6 @@ import ( e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" ) -const etcdImage = "3.5.5-0" - -// EtcdUpgrade upgrades etcd on GCE. -func EtcdUpgrade(targetStorage, targetVersion string) error { - switch TestContext.Provider { - case "gce": - return etcdUpgradeGCE(targetStorage, targetVersion) - default: - return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider) - } -} - -func etcdUpgradeGCE(targetStorage, targetVersion string) error { - env := append( - os.Environ(), - "TEST_ETCD_VERSION="+targetVersion, - "STORAGE_BACKEND="+targetStorage, - "TEST_ETCD_IMAGE="+etcdImage) - - _, _, err := RunCmdEnv(env, GCEUpgradeScript(), "-l", "-M") - return err -} - -// LocationParamGKE returns parameter related to location for gcloud command. -func LocationParamGKE() string { - if TestContext.CloudConfig.MultiMaster { - // GKE Regional Clusters are being tested. - return fmt.Sprintf("--region=%s", TestContext.CloudConfig.Region) - } - return fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone) -} - // AppendContainerCommandGroupIfNeeded returns container command group parameter if necessary. func AppendContainerCommandGroupIfNeeded(args []string) []string { if TestContext.CloudConfig.Region != "" { @@ -75,55 +40,6 @@ func AppendContainerCommandGroupIfNeeded(args []string) []string { return args } -// MasterUpgradeGKE upgrades master node to the specified version on GKE. -func MasterUpgradeGKE(namespace string, v string) error { - Logf("Upgrading master to %q", v) - args := []string{ - "container", - "clusters", - fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID), - LocationParamGKE(), - "upgrade", - TestContext.CloudConfig.Cluster, - "--master", - fmt.Sprintf("--cluster-version=%s", v), - "--quiet", - } - _, _, err := RunCmd("gcloud", AppendContainerCommandGroupIfNeeded(args)...) - if err != nil { - return err - } - - WaitForSSHTunnels(namespace) - - return nil -} - -// GCEUpgradeScript returns path of script for upgrading on GCE. -func GCEUpgradeScript() string { - if len(TestContext.GCEUpgradeScript) == 0 { - return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh") - } - return TestContext.GCEUpgradeScript -} - -// WaitForSSHTunnels waits for establishing SSH tunnel to busybox pod. -func WaitForSSHTunnels(namespace string) { - Logf("Waiting for SSH tunnels to establish") - RunKubectl(namespace, "run", "ssh-tunnel-test", - "--image=busybox", - "--restart=Never", - "--command", "--", - "echo", "Hello") - defer RunKubectl(namespace, "delete", "pod", "ssh-tunnel-test") - - // allow up to a minute for new ssh tunnels to establish - wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) { - _, err := RunKubectl(namespace, "logs", "ssh-tunnel-test") - return err == nil, nil - }) -} - // NodeKiller is a utility to simulate node failures. type NodeKiller struct { config NodeKillerConfig diff --git a/test/e2e/framework/service/jig.go b/test/e2e/framework/service/jig.go index 5561030e753..8fc99bc8a9c 100644 --- a/test/e2e/framework/service/jig.go +++ b/test/e2e/framework/service/jig.go @@ -46,6 +46,7 @@ import ( e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2erc "k8s.io/kubernetes/test/e2e/framework/rc" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" netutils "k8s.io/utils/net" @@ -910,7 +911,7 @@ func testEndpointReachability(endpoint string, port int32, protocol v1.Protocol, } err := wait.PollImmediate(1*time.Second, ServiceReachabilityShortPollTimeout, func() (bool, error) { - _, err := framework.RunHostCmd(execPod.Namespace, execPod.Name, cmd) + _, err := e2etodopod.RunHostCmd(execPod.Namespace, execPod.Name, cmd) if err != nil { framework.Logf("Service reachability failing with error: %v\nRetrying...", err) return false, nil @@ -1002,7 +1003,7 @@ func (j *TestJig) checkExternalServiceReachability(svc *v1.Service, pod *v1.Pod) // Service must resolve to IP cmd := fmt.Sprintf("nslookup %s", svcName) return wait.PollImmediate(framework.Poll, ServiceReachabilityShortPollTimeout, func() (done bool, err error) { - _, stderr, err := framework.RunHostCmdWithFullOutput(pod.Namespace, pod.Name, cmd) + _, stderr, err := e2etodopod.RunHostCmdWithFullOutput(pod.Namespace, pod.Name, cmd) // NOTE(claudiub): nslookup may return 0 on Windows, even though the DNS name was not found. In this case, // we can check stderr for the error. if err != nil || (framework.NodeOSDistroIs("windows") && strings.Contains(stderr, fmt.Sprintf("can't find %s", svcName))) { diff --git a/test/e2e/framework/statefulset/fixtures.go b/test/e2e/framework/statefulset/fixtures.go index 375d20051c5..887b3e617f7 100644 --- a/test/e2e/framework/statefulset/fixtures.go +++ b/test/e2e/framework/statefulset/fixtures.go @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubectl/pkg/util/podutils" "k8s.io/kubernetes/test/e2e/framework" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -165,7 +166,7 @@ func ResumeNextPod(c clientset.Interface, ss *appsv1.StatefulSet) { if resumedPod != "" { framework.Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod) } - _, err := framework.RunHostCmdWithRetries(pod.Namespace, pod.Name, "dd if=/dev/zero of=/data/statefulset-continue bs=1 count=1 conv=fsync", StatefulSetPoll, StatefulPodTimeout) + _, err := e2etodopod.RunHostCmdWithRetries(pod.Namespace, pod.Name, "dd if=/dev/zero of=/data/statefulset-continue bs=1 count=1 conv=fsync", StatefulSetPoll, StatefulPodTimeout) framework.ExpectNoError(err) framework.Logf("Resumed pod %v", pod.Name) resumedPod = pod.Name diff --git a/test/e2e/framework/statefulset/rest.go b/test/e2e/framework/statefulset/rest.go index 65aaac0cb66..e7c7187cdc1 100644 --- a/test/e2e/framework/statefulset/rest.go +++ b/test/e2e/framework/statefulset/rest.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubectl/pkg/util/podutils" "k8s.io/kubernetes/test/e2e/framework" e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" ) // CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the Namespace ns using kubectl create. @@ -192,7 +193,7 @@ func CheckHostname(c clientset.Interface, ss *appsv1.StatefulSet) error { cmd := "printf $(hostname)" podList := GetPodList(c, ss) for _, statefulPod := range podList.Items { - hostname, err := framework.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout) + hostname, err := e2etodopod.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout) if err != nil { return err } @@ -236,7 +237,7 @@ func CheckServiceName(ss *appsv1.StatefulSet, expectedServiceName string) error func ExecInStatefulPods(c clientset.Interface, ss *appsv1.StatefulSet, cmd string) error { podList := GetPodList(c, ss) for _, statefulPod := range podList.Items { - stdout, err := framework.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout) + stdout, err := e2etodopod.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout) framework.Logf("stdout of %v on %v: %v", cmd, statefulPod.Name, stdout) if err != nil { return err diff --git a/test/e2e/framework/todo/kubectl/builder.go b/test/e2e/framework/todo/kubectl/builder.go new file mode 100644 index 00000000000..a11fe6394e7 --- /dev/null +++ b/test/e2e/framework/todo/kubectl/builder.go @@ -0,0 +1,192 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "bytes" + "fmt" + "io" + "net" + "net/url" + "os/exec" + "strings" + "syscall" + "time" + + "k8s.io/client-go/tools/clientcmd" + uexec "k8s.io/utils/exec" + + "k8s.io/kubernetes/test/e2e/framework" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" +) + +// KubectlBuilder is used to build, customize and execute a kubectl Command. +// Add more functions to customize the builder as needed. +type KubectlBuilder struct { + cmd *exec.Cmd + timeout <-chan time.Time +} + +// NewKubectlCommand returns a KubectlBuilder for running kubectl. +func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder { + b := new(KubectlBuilder) + tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, namespace) + b.cmd = tk.KubectlCmd(args...) + return b +} + +// WithEnv sets the given environment and returns itself. +func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder { + b.cmd.Env = env + return b +} + +// WithTimeout sets the given timeout and returns itself. +func (b *KubectlBuilder) WithTimeout(t <-chan time.Time) *KubectlBuilder { + b.timeout = t + return b +} + +// WithStdinData sets the given data to stdin and returns itself. +func (b KubectlBuilder) WithStdinData(data string) *KubectlBuilder { + b.cmd.Stdin = strings.NewReader(data) + return &b +} + +// WithStdinReader sets the given reader and returns itself. +func (b KubectlBuilder) WithStdinReader(reader io.Reader) *KubectlBuilder { + b.cmd.Stdin = reader + return &b +} + +// ExecOrDie runs the kubectl executable or dies if error occurs. +func (b KubectlBuilder) ExecOrDie(namespace string) string { + str, err := b.Exec() + // In case of i/o timeout error, try talking to the apiserver again after 2s before dying. + // Note that we're still dying after retrying so that we can get visibility to triage it further. + if isTimeout(err) { + framework.Logf("Hit i/o timeout error, talking to the server 2s later to see if it's temporary.") + time.Sleep(2 * time.Second) + retryStr, retryErr := RunKubectl(namespace, "version") + framework.Logf("stdout: %q", retryStr) + framework.Logf("err: %v", retryErr) + } + framework.ExpectNoError(err) + return str +} + +func isTimeout(err error) bool { + switch err := err.(type) { + case *url.Error: + if err, ok := err.Err.(net.Error); ok && err.Timeout() { + return true + } + case net.Error: + if err.Timeout() { + return true + } + } + return false +} + +// Exec runs the kubectl executable. +func (b KubectlBuilder) Exec() (string, error) { + stdout, _, err := b.ExecWithFullOutput() + return stdout, err +} + +// ExecWithFullOutput runs the kubectl executable, and returns the stdout and stderr. +func (b KubectlBuilder) ExecWithFullOutput() (string, string, error) { + var stdout, stderr bytes.Buffer + cmd := b.cmd + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + framework.Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately + if err := cmd.Start(); err != nil { + return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err) + } + errCh := make(chan error, 1) + go func() { + errCh <- cmd.Wait() + }() + select { + case err := <-errCh: + if err != nil { + var rc = 127 + if ee, ok := err.(*exec.ExitError); ok { + rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus()) + framework.Logf("rc: %d", rc) + } + return stdout.String(), stderr.String(), uexec.CodeExitError{ + Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err), + Code: rc, + } + } + case <-b.timeout: + b.cmd.Process.Kill() + return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr) + } + framework.Logf("stderr: %q", stderr.String()) + framework.Logf("stdout: %q", stdout.String()) + return stdout.String(), stderr.String(), nil +} + +// RunKubectlOrDie is a convenience wrapper over kubectlBuilder +func RunKubectlOrDie(namespace string, args ...string) string { + return NewKubectlCommand(namespace, args...).ExecOrDie(namespace) +} + +// RunKubectl is a convenience wrapper over kubectlBuilder +func RunKubectl(namespace string, args ...string) (string, error) { + return NewKubectlCommand(namespace, args...).Exec() +} + +// RunKubectlWithFullOutput is a convenience wrapper over kubectlBuilder +// It will also return the command's stderr. +func RunKubectlWithFullOutput(namespace string, args ...string) (string, string, error) { + return NewKubectlCommand(namespace, args...).ExecWithFullOutput() +} + +// RunKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin +func RunKubectlOrDieInput(namespace string, data string, args ...string) string { + return NewKubectlCommand(namespace, args...).WithStdinData(data).ExecOrDie(namespace) +} + +// RunKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin +func RunKubectlInput(namespace string, data string, args ...string) (string, error) { + return NewKubectlCommand(namespace, args...).WithStdinData(data).Exec() +} + +// RunKubemciWithKubeconfig is a convenience wrapper over RunKubemciCmd +func RunKubemciWithKubeconfig(args ...string) (string, error) { + if framework.TestContext.KubeConfig != "" { + args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+framework.TestContext.KubeConfig) + } + return RunKubemciCmd(args...) +} + +// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci. +// It assumes that kubemci exists in PATH. +func RunKubemciCmd(args ...string) (string, error) { + // kubemci is assumed to be in PATH. + kubemci := "kubemci" + b := new(KubectlBuilder) + args = append(args, "--gcp-project="+framework.TestContext.CloudConfig.ProjectID) + + b.cmd = exec.Command(kubemci, args...) + return b.Exec() +} diff --git a/test/e2e/framework/todo/node/ssh.go b/test/e2e/framework/todo/node/ssh.go new file mode 100644 index 00000000000..f2ec508bb48 --- /dev/null +++ b/test/e2e/framework/todo/node/ssh.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/wait" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl" +) + +// WaitForSSHTunnels waits for establishing SSH tunnel to busybox pod. +func WaitForSSHTunnels(namespace string) { + e2elog.Logf("Waiting for SSH tunnels to establish") + e2etodokubectl.RunKubectl(namespace, "run", "ssh-tunnel-test", + "--image=busybox", + "--restart=Never", + "--command", "--", + "echo", "Hello") + defer e2etodokubectl.RunKubectl(namespace, "delete", "pod", "ssh-tunnel-test") + + // allow up to a minute for new ssh tunnels to establish + wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) { + _, err := e2etodokubectl.RunKubectl(namespace, "logs", "ssh-tunnel-test") + return err == nil, nil + }) +} diff --git a/test/e2e/framework/todo/pod/output.go b/test/e2e/framework/todo/pod/output.go new file mode 100644 index 00000000000..dd24b7b8e4f --- /dev/null +++ b/test/e2e/framework/todo/pod/output.go @@ -0,0 +1,238 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" + apiv1pod "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl" +) + +// DEPRECATED constants. Use the timeouts in framework.Framework instead. +const ( + // Poll is how often to Poll pods, nodes and claims. + Poll = 2 * time.Second +) + +// LookForStringInPodExec looks for the given string in the output of a command +// executed in the first container of specified pod. +func LookForStringInPodExec(ns, podName string, command []string, expectedString string, timeout time.Duration) (result string, err error) { + return LookForStringInPodExecToContainer(ns, podName, "", command, expectedString, timeout) +} + +// LookForStringInPodExecToContainer looks for the given string in the output of a +// command executed in specified pod container, or first container if not specified. +func LookForStringInPodExecToContainer(ns, podName, containerName string, command []string, expectedString string, timeout time.Duration) (result string, err error) { + return lookForString(expectedString, timeout, func() string { + args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns)} + if len(containerName) > 0 { + args = append(args, fmt.Sprintf("--container=%s", containerName)) + } + args = append(args, "--") + args = append(args, command...) + return e2etodokubectl.RunKubectlOrDie(ns, args...) + }) +} + +// lookForString looks for the given string in the output of fn, repeatedly calling fn until +// the timeout is reached or the string is found. Returns last log and possibly +// error if the string was not found. +func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) { + for t := time.Now(); time.Since(t) < timeout; time.Sleep(Poll) { + result = fn() + if strings.Contains(result, expectedString) { + return + } + } + err = fmt.Errorf("Failed to find \"%s\", last result: \"%s\"", expectedString, result) + return +} + +// RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec` +// inside of a shell. +func RunHostCmd(ns, name, cmd string) (string, error) { + return e2etodokubectl.RunKubectl(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd) +} + +// RunHostCmdWithFullOutput runs the given cmd in the context of the given pod using `kubectl exec` +// inside of a shell. It will also return the command's stderr. +func RunHostCmdWithFullOutput(ns, name, cmd string) (string, string, error) { + return e2etodokubectl.RunKubectlWithFullOutput(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd) +} + +// RunHostCmdOrDie calls RunHostCmd and dies on error. +func RunHostCmdOrDie(ns, name, cmd string) string { + stdout, err := RunHostCmd(ns, name, cmd) + framework.Logf("stdout: %v", stdout) + framework.ExpectNoError(err) + return stdout +} + +// RunHostCmdWithRetries calls RunHostCmd and retries all errors +// until it succeeds or the specified timeout expires. +// This can be used with idempotent commands to deflake transient Node issues. +func RunHostCmdWithRetries(ns, name, cmd string, interval, timeout time.Duration) (string, error) { + start := time.Now() + for { + out, err := RunHostCmd(ns, name, cmd) + if err == nil { + return out, nil + } + if elapsed := time.Since(start); elapsed > timeout { + return out, fmt.Errorf("RunHostCmd still failed after %v: %v", elapsed, err) + } + framework.Logf("Waiting %v to retry failed RunHostCmd: %v", interval, err) + time.Sleep(interval) + } +} + +// LookForStringInLog looks for the given string in the log of a specific pod container +func LookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) { + return lookForString(expectedString, timeout, func() string { + return e2etodokubectl.RunKubectlOrDie(ns, "logs", podName, container) + }) +} + +// CreateEmptyFileOnPod creates empty file at given path on the pod. +func CreateEmptyFileOnPod(namespace string, podName string, filePath string) error { + _, err := e2etodokubectl.RunKubectl(namespace, "exec", podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath)) + return err +} + +// DumpDebugInfo dumps debug info of tests. +func DumpDebugInfo(c clientset.Interface, ns string) { + sl, _ := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.Everything().String()}) + for _, s := range sl.Items { + desc, _ := e2etodokubectl.RunKubectl(ns, "describe", "po", s.Name) + framework.Logf("\nOutput of kubectl describe %v:\n%v", s.Name, desc) + + l, _ := e2etodokubectl.RunKubectl(ns, "logs", s.Name, "--tail=100") + framework.Logf("\nLast 100 log lines of %v:\n%v", s.Name, l) + } +} + +// MatchContainerOutput creates a pod and waits for all it's containers to exit with success. +// It then tests that the matcher with each expectedOutput matches the output of the specified container. +func MatchContainerOutput( + f *framework.Framework, + pod *v1.Pod, + containerName string, + expectedOutput []string, + matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error { + ns := pod.ObjectMeta.Namespace + if ns == "" { + ns = f.Namespace.Name + } + podClient := f.PodClientNS(ns) + + createdPod := podClient.Create(pod) + defer func() { + ginkgo.By("delete the pod") + podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) + }() + + // Wait for client pod to complete. + podErr := e2epod.WaitForPodSuccessInNamespaceTimeout(f.ClientSet, createdPod.Name, ns, f.Timeouts.PodStart) + + // Grab its logs. Get host first. + podStatus, err := podClient.Get(context.TODO(), createdPod.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get pod status: %v", err) + } + + if podErr != nil { + // Pod failed. Dump all logs from all containers to see what's wrong + _ = apiv1pod.VisitContainers(&podStatus.Spec, apiv1pod.AllFeatureEnabledContainers(), func(c *v1.Container, containerType apiv1pod.ContainerType) bool { + logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, c.Name) + if err != nil { + framework.Logf("Failed to get logs from node %q pod %q container %q: %v", + podStatus.Spec.NodeName, podStatus.Name, c.Name, err) + } else { + framework.Logf("Output of node %q pod %q container %q: %s", podStatus.Spec.NodeName, podStatus.Name, c.Name, logs) + } + return true + }) + return fmt.Errorf("expected pod %q success: %v", createdPod.Name, podErr) + } + + framework.Logf("Trying to get logs from node %s pod %s container %s: %v", + podStatus.Spec.NodeName, podStatus.Name, containerName, err) + + // Sometimes the actual containers take a second to get started, try to get logs for 60s + logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, containerName) + if err != nil { + framework.Logf("Failed to get logs from node %q pod %q container %q. %v", + podStatus.Spec.NodeName, podStatus.Name, containerName, err) + return fmt.Errorf("failed to get logs from %s for %s: %v", podStatus.Name, containerName, err) + } + + for _, expected := range expectedOutput { + m := matcher(expected) + matches, err := m.Match(logs) + if err != nil { + return fmt.Errorf("expected %q in container output: %v", expected, err) + } else if !matches { + return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs)) + } + } + + return nil +} + +// TestContainerOutput runs the given pod in the given namespace and waits +// for all of the containers in the podSpec to move into the 'Success' status, and tests +// the specified container log against the given expected output using a substring matcher. +func TestContainerOutput(f *framework.Framework, scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) { + TestContainerOutputMatcher(f, scenarioName, pod, containerIndex, expectedOutput, gomega.ContainSubstring) +} + +// TestContainerOutputRegexp runs the given pod in the given namespace and waits +// for all of the containers in the podSpec to move into the 'Success' status, and tests +// the specified container log against the given expected output using a regexp matcher. +func TestContainerOutputRegexp(f *framework.Framework, scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) { + TestContainerOutputMatcher(f, scenarioName, pod, containerIndex, expectedOutput, gomega.MatchRegexp) +} + +// TestContainerOutputMatcher runs the given pod in the given namespace and waits +// for all of the containers in the podSpec to move into the 'Success' status, and tests +// the specified container log against the given expected output using the given matcher. +func TestContainerOutputMatcher(f *framework.Framework, + scenarioName string, + pod *v1.Pod, + containerIndex int, + expectedOutput []string, + matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) { + ginkgo.By(fmt.Sprintf("Creating a pod to test %v", scenarioName)) + if containerIndex < 0 || containerIndex >= len(pod.Spec.Containers) { + framework.Failf("Invalid container index: %d", containerIndex) + } + framework.ExpectNoError(MatchContainerOutput(f, pod, pod.Spec.Containers[containerIndex].Name, expectedOutput, matcher)) +} diff --git a/test/e2e/framework/todo/providers/gcp.go b/test/e2e/framework/todo/providers/gcp.go new file mode 100644 index 00000000000..de52530ac56 --- /dev/null +++ b/test/e2e/framework/todo/providers/gcp.go @@ -0,0 +1,90 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "os" + "path" + + "k8s.io/kubernetes/test/e2e/framework" + e2etodonode "k8s.io/kubernetes/test/e2e/framework/todo/node" +) + +const etcdImage = "3.5.5-0" + +// EtcdUpgrade upgrades etcd on GCE. +func EtcdUpgrade(targetStorage, targetVersion string) error { + switch framework.TestContext.Provider { + case "gce": + return etcdUpgradeGCE(targetStorage, targetVersion) + default: + return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", framework.TestContext.Provider) + } +} + +func etcdUpgradeGCE(targetStorage, targetVersion string) error { + env := append( + os.Environ(), + "TEST_ETCD_VERSION="+targetVersion, + "STORAGE_BACKEND="+targetStorage, + "TEST_ETCD_IMAGE="+etcdImage) + + _, _, err := framework.RunCmdEnv(env, GCEUpgradeScript(), "-l", "-M") + return err +} + +// LocationParamGKE returns parameter related to location for gcloud command. +func LocationParamGKE() string { + if framework.TestContext.CloudConfig.MultiMaster { + // GKE Regional Clusters are being tested. + return fmt.Sprintf("--region=%s", framework.TestContext.CloudConfig.Region) + } + return fmt.Sprintf("--zone=%s", framework.TestContext.CloudConfig.Zone) +} + +// MasterUpgradeGKE upgrades master node to the specified version on GKE. +func MasterUpgradeGKE(namespace string, v string) error { + framework.Logf("Upgrading master to %q", v) + args := []string{ + "container", + "clusters", + fmt.Sprintf("--project=%s", framework.TestContext.CloudConfig.ProjectID), + LocationParamGKE(), + "upgrade", + framework.TestContext.CloudConfig.Cluster, + "--master", + fmt.Sprintf("--cluster-version=%s", v), + "--quiet", + } + _, _, err := framework.RunCmd("gcloud", framework.AppendContainerCommandGroupIfNeeded(args)...) + if err != nil { + return err + } + + e2etodonode.WaitForSSHTunnels(namespace) + + return nil +} + +// GCEUpgradeScript returns path of script for upgrading on GCE. +func GCEUpgradeScript() string { + if len(framework.TestContext.GCEUpgradeScript) == 0 { + return path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh") + } + return framework.TestContext.GCEUpgradeScript +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 57166938473..02541589d22 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -23,7 +23,6 @@ import ( "fmt" "io" "math/rand" - "net" "net/url" "os" "os/exec" @@ -32,18 +31,15 @@ import ( "strconv" "strings" "sync" - "syscall" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - gomegatypes "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -59,11 +55,10 @@ import ( watchtools "k8s.io/client-go/tools/watch" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" - uexec "k8s.io/utils/exec" netutils "k8s.io/utils/net" // TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245) - e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -75,10 +70,6 @@ const ( // TODO(justinsb): Avoid hardcoding this. awsMasterIP = "172.20.0.9" - - // AllContainers specifies that all containers be visited - // Copied from pkg/api/v1/pod to avoid pulling extra dependencies - AllContainers = InitContainers | Containers | EphemeralContainers ) // DEPRECATED constants. Use the timeouts in framework.Framework instead. @@ -543,199 +534,6 @@ func RandomSuffix() string { return strconv.Itoa(rand.Intn(10000)) } -// LookForStringInPodExec looks for the given string in the output of a command -// executed in the first container of specified pod. -// TODO(alejandrox1): move to pod/ subpkg once kubectl methods are refactored. -func LookForStringInPodExec(ns, podName string, command []string, expectedString string, timeout time.Duration) (result string, err error) { - return LookForStringInPodExecToContainer(ns, podName, "", command, expectedString, timeout) -} - -// LookForStringInPodExecToContainer looks for the given string in the output of a -// command executed in specified pod container, or first container if not specified. -func LookForStringInPodExecToContainer(ns, podName, containerName string, command []string, expectedString string, timeout time.Duration) (result string, err error) { - return lookForString(expectedString, timeout, func() string { - args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns)} - if len(containerName) > 0 { - args = append(args, fmt.Sprintf("--container=%s", containerName)) - } - args = append(args, "--") - args = append(args, command...) - return RunKubectlOrDie(ns, args...) - }) -} - -// lookForString looks for the given string in the output of fn, repeatedly calling fn until -// the timeout is reached or the string is found. Returns last log and possibly -// error if the string was not found. -// TODO(alejandrox1): move to pod/ subpkg once kubectl methods are refactored. -func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) { - for t := time.Now(); time.Since(t) < timeout; time.Sleep(Poll) { - result = fn() - if strings.Contains(result, expectedString) { - return - } - } - err = fmt.Errorf("Failed to find \"%s\", last result: \"%s\"", expectedString, result) - return -} - -// KubectlBuilder is used to build, customize and execute a kubectl Command. -// Add more functions to customize the builder as needed. -type KubectlBuilder struct { - cmd *exec.Cmd - timeout <-chan time.Time -} - -// NewKubectlCommand returns a KubectlBuilder for running kubectl. -func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder { - b := new(KubectlBuilder) - tk := e2ekubectl.NewTestKubeconfig(TestContext.CertDir, TestContext.Host, TestContext.KubeConfig, TestContext.KubeContext, TestContext.KubectlPath, namespace) - b.cmd = tk.KubectlCmd(args...) - return b -} - -// WithEnv sets the given environment and returns itself. -func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder { - b.cmd.Env = env - return b -} - -// WithTimeout sets the given timeout and returns itself. -func (b *KubectlBuilder) WithTimeout(t <-chan time.Time) *KubectlBuilder { - b.timeout = t - return b -} - -// WithStdinData sets the given data to stdin and returns itself. -func (b KubectlBuilder) WithStdinData(data string) *KubectlBuilder { - b.cmd.Stdin = strings.NewReader(data) - return &b -} - -// WithStdinReader sets the given reader and returns itself. -func (b KubectlBuilder) WithStdinReader(reader io.Reader) *KubectlBuilder { - b.cmd.Stdin = reader - return &b -} - -// ExecOrDie runs the kubectl executable or dies if error occurs. -func (b KubectlBuilder) ExecOrDie(namespace string) string { - str, err := b.Exec() - // In case of i/o timeout error, try talking to the apiserver again after 2s before dying. - // Note that we're still dying after retrying so that we can get visibility to triage it further. - if isTimeout(err) { - Logf("Hit i/o timeout error, talking to the server 2s later to see if it's temporary.") - time.Sleep(2 * time.Second) - retryStr, retryErr := RunKubectl(namespace, "version") - Logf("stdout: %q", retryStr) - Logf("err: %v", retryErr) - } - ExpectNoError(err) - return str -} - -func isTimeout(err error) bool { - switch err := err.(type) { - case *url.Error: - if err, ok := err.Err.(net.Error); ok && err.Timeout() { - return true - } - case net.Error: - if err.Timeout() { - return true - } - } - return false -} - -// Exec runs the kubectl executable. -func (b KubectlBuilder) Exec() (string, error) { - stdout, _, err := b.ExecWithFullOutput() - return stdout, err -} - -// ExecWithFullOutput runs the kubectl executable, and returns the stdout and stderr. -func (b KubectlBuilder) ExecWithFullOutput() (string, string, error) { - var stdout, stderr bytes.Buffer - cmd := b.cmd - cmd.Stdout, cmd.Stderr = &stdout, &stderr - - Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately - if err := cmd.Start(); err != nil { - return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err) - } - errCh := make(chan error, 1) - go func() { - errCh <- cmd.Wait() - }() - select { - case err := <-errCh: - if err != nil { - var rc = 127 - if ee, ok := err.(*exec.ExitError); ok { - rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus()) - Logf("rc: %d", rc) - } - return stdout.String(), stderr.String(), uexec.CodeExitError{ - Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err), - Code: rc, - } - } - case <-b.timeout: - b.cmd.Process.Kill() - return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr) - } - Logf("stderr: %q", stderr.String()) - Logf("stdout: %q", stdout.String()) - return stdout.String(), stderr.String(), nil -} - -// RunKubectlOrDie is a convenience wrapper over kubectlBuilder -func RunKubectlOrDie(namespace string, args ...string) string { - return NewKubectlCommand(namespace, args...).ExecOrDie(namespace) -} - -// RunKubectl is a convenience wrapper over kubectlBuilder -func RunKubectl(namespace string, args ...string) (string, error) { - return NewKubectlCommand(namespace, args...).Exec() -} - -// RunKubectlWithFullOutput is a convenience wrapper over kubectlBuilder -// It will also return the command's stderr. -func RunKubectlWithFullOutput(namespace string, args ...string) (string, string, error) { - return NewKubectlCommand(namespace, args...).ExecWithFullOutput() -} - -// RunKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin -func RunKubectlOrDieInput(namespace string, data string, args ...string) string { - return NewKubectlCommand(namespace, args...).WithStdinData(data).ExecOrDie(namespace) -} - -// RunKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin -func RunKubectlInput(namespace string, data string, args ...string) (string, error) { - return NewKubectlCommand(namespace, args...).WithStdinData(data).Exec() -} - -// RunKubemciWithKubeconfig is a convenience wrapper over RunKubemciCmd -func RunKubemciWithKubeconfig(args ...string) (string, error) { - if TestContext.KubeConfig != "" { - args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig) - } - return RunKubemciCmd(args...) -} - -// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci. -// It assumes that kubemci exists in PATH. -func RunKubemciCmd(args ...string) (string, error) { - // kubemci is assumed to be in PATH. - kubemci := "kubemci" - b := new(KubectlBuilder) - args = append(args, "--gcp-project="+TestContext.CloudConfig.ProjectID) - - b.cmd = exec.Command(kubemci, args...) - return b.Exec() -} - // StartCmdAndStreamOutput returns stdout and stderr after starting the given cmd. func StartCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { stdout, err = cmd.StdoutPipe() @@ -758,142 +556,6 @@ func TryKill(cmd *exec.Cmd) { } } -// testContainerOutputMatcher runs the given pod in the given namespace and waits -// for all of the containers in the podSpec to move into the 'Success' status, and tests -// the specified container log against the given expected output using the given matcher. -func (f *Framework) testContainerOutputMatcher(scenarioName string, - pod *v1.Pod, - containerIndex int, - expectedOutput []string, - matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) { - ginkgo.By(fmt.Sprintf("Creating a pod to test %v", scenarioName)) - if containerIndex < 0 || containerIndex >= len(pod.Spec.Containers) { - Failf("Invalid container index: %d", containerIndex) - } - ExpectNoError(f.MatchContainerOutput(pod, pod.Spec.Containers[containerIndex].Name, expectedOutput, matcher)) -} - -// ContainerType signifies container type -type ContainerType int - -const ( - // Containers is for normal containers - Containers ContainerType = 1 << iota - // InitContainers is for init containers - InitContainers - // EphemeralContainers is for ephemeral containers - EphemeralContainers -) - -// allFeatureEnabledContainers returns a ContainerType mask which includes all container -// types except for the ones guarded by feature gate. -// Copied from pkg/api/v1/pod to avoid pulling extra dependencies -func allFeatureEnabledContainers() ContainerType { - return AllContainers -} - -// ContainerVisitor is called with each container spec, and returns true -// if visiting should continue. -// Copied from pkg/api/v1/pod to avoid pulling extra dependencies -type ContainerVisitor func(container *v1.Container, containerType ContainerType) (shouldContinue bool) - -// visitContainers invokes the visitor function with a pointer to every container -// spec in the given pod spec with type set in mask. If visitor returns false, -// visiting is short-circuited. visitContainers returns true if visiting completes, -// false if visiting was short-circuited. -// Copied from pkg/api/v1/pod to avoid pulling extra dependencies -func visitContainers(podSpec *v1.PodSpec, mask ContainerType, visitor ContainerVisitor) bool { - if mask&InitContainers != 0 { - for i := range podSpec.InitContainers { - if !visitor(&podSpec.InitContainers[i], InitContainers) { - return false - } - } - } - if mask&Containers != 0 { - for i := range podSpec.Containers { - if !visitor(&podSpec.Containers[i], Containers) { - return false - } - } - } - if mask&EphemeralContainers != 0 { - for i := range podSpec.EphemeralContainers { - if !visitor((*v1.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon), EphemeralContainers) { - return false - } - } - } - return true -} - -// MatchContainerOutput creates a pod and waits for all it's containers to exit with success. -// It then tests that the matcher with each expectedOutput matches the output of the specified container. -func (f *Framework) MatchContainerOutput( - pod *v1.Pod, - containerName string, - expectedOutput []string, - matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error { - ns := pod.ObjectMeta.Namespace - if ns == "" { - ns = f.Namespace.Name - } - podClient := f.PodClientNS(ns) - - createdPod := podClient.Create(pod) - defer func() { - ginkgo.By("delete the pod") - podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, DefaultPodDeletionTimeout) - }() - - // Wait for client pod to complete. - podErr := e2epod.WaitForPodSuccessInNamespaceTimeout(f.ClientSet, createdPod.Name, ns, f.Timeouts.PodStart) - - // Grab its logs. Get host first. - podStatus, err := podClient.Get(context.TODO(), createdPod.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get pod status: %v", err) - } - - if podErr != nil { - // Pod failed. Dump all logs from all containers to see what's wrong - _ = visitContainers(&podStatus.Spec, allFeatureEnabledContainers(), func(c *v1.Container, containerType ContainerType) bool { - logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, c.Name) - if err != nil { - Logf("Failed to get logs from node %q pod %q container %q: %v", - podStatus.Spec.NodeName, podStatus.Name, c.Name, err) - } else { - Logf("Output of node %q pod %q container %q: %s", podStatus.Spec.NodeName, podStatus.Name, c.Name, logs) - } - return true - }) - return fmt.Errorf("expected pod %q success: %v", createdPod.Name, podErr) - } - - Logf("Trying to get logs from node %s pod %s container %s: %v", - podStatus.Spec.NodeName, podStatus.Name, containerName, err) - - // Sometimes the actual containers take a second to get started, try to get logs for 60s - logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, containerName) - if err != nil { - Logf("Failed to get logs from node %q pod %q container %q. %v", - podStatus.Spec.NodeName, podStatus.Name, containerName, err) - return fmt.Errorf("failed to get logs from %s for %s: %v", podStatus.Name, containerName, err) - } - - for _, expected := range expectedOutput { - m := matcher(expected) - matches, err := m.Match(logs) - if err != nil { - return fmt.Errorf("expected %q in container output: %v", expected, err) - } else if !matches { - return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs)) - } - } - - return nil -} - // EventsLister is a func that lists events. type EventsLister func(opts metav1.ListOptions, ns string) (*v1.EventList, error) @@ -1108,44 +770,6 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool return true, nil } -// RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec` -// inside of a shell. -func RunHostCmd(ns, name, cmd string) (string, error) { - return RunKubectl(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd) -} - -// RunHostCmdWithFullOutput runs the given cmd in the context of the given pod using `kubectl exec` -// inside of a shell. It will also return the command's stderr. -func RunHostCmdWithFullOutput(ns, name, cmd string) (string, string, error) { - return RunKubectlWithFullOutput(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd) -} - -// RunHostCmdOrDie calls RunHostCmd and dies on error. -func RunHostCmdOrDie(ns, name, cmd string) string { - stdout, err := RunHostCmd(ns, name, cmd) - Logf("stdout: %v", stdout) - ExpectNoError(err) - return stdout -} - -// RunHostCmdWithRetries calls RunHostCmd and retries all errors -// until it succeeds or the specified timeout expires. -// This can be used with idempotent commands to deflake transient Node issues. -func RunHostCmdWithRetries(ns, name, cmd string, interval, timeout time.Duration) (string, error) { - start := time.Now() - for { - out, err := RunHostCmd(ns, name, cmd) - if err == nil { - return out, nil - } - if elapsed := time.Since(start); elapsed > timeout { - return out, fmt.Errorf("RunHostCmd still failed after %v: %v", elapsed, err) - } - Logf("Waiting %v to retry failed RunHostCmd: %v", interval, err) - time.Sleep(interval) - } -} - // AllNodesReady checks whether all registered nodes are ready. Setting -1 on // TestContext.AllowedNotReadyNodes will bypass the post test node readiness check. // TODO: we should change the AllNodesReady call in AfterEach to WaitForAllNodesHealthy, @@ -1194,13 +818,6 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error { return nil } -// LookForStringInLog looks for the given string in the log of a specific pod container -func LookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) { - return lookForString(expectedString, timeout, func() string { - return RunKubectlOrDie(ns, "logs", podName, container) - }) -} - // EnsureLoadBalancerResourcesDeleted ensures that cloud load balancer resources that were created // are actually cleaned up. Currently only implemented for GCE/GKE. func EnsureLoadBalancerResourcesDeleted(ip, portRange string) error { @@ -1323,25 +940,6 @@ func GetControlPlaneAddresses(c clientset.Interface) []string { return ips.List() } -// CreateEmptyFileOnPod creates empty file at given path on the pod. -// TODO(alejandrox1): move to subpkg pod once kubectl methods have been refactored. -func CreateEmptyFileOnPod(namespace string, podName string, filePath string) error { - _, err := RunKubectl(namespace, "exec", podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath)) - return err -} - -// DumpDebugInfo dumps debug info of tests. -func DumpDebugInfo(c clientset.Interface, ns string) { - sl, _ := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.Everything().String()}) - for _, s := range sl.Items { - desc, _ := RunKubectl(ns, "describe", "po", s.Name) - Logf("\nOutput of kubectl describe %v:\n%v", s.Name, desc) - - l, _ := RunKubectl(ns, "logs", s.Name, "--tail=100") - Logf("\nLast 100 log lines of %v:\n%v", s.Name, l) - } -} - // PrettyPrintJSON converts metrics to JSON format. func PrettyPrintJSON(metrics interface{}) string { output := &bytes.Buffer{} diff --git a/test/e2e/framework/volume/fixtures.go b/test/e2e/framework/volume/fixtures.go index 15c1b654438..beaf4f6fa7e 100644 --- a/test/e2e/framework/volume/fixtures.go +++ b/test/e2e/framework/volume/fixtures.go @@ -57,6 +57,8 @@ import ( clientexec "k8s.io/client-go/util/exec" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl" + e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod" imageutils "k8s.io/kubernetes/test/utils/image" uexec "k8s.io/utils/exec" @@ -354,7 +356,7 @@ func startVolumeServer(client clientset.Interface, config TestConfig) *v1.Pod { } } if config.ServerReadyMessage != "" { - _, err := framework.LookForStringInLog(pod.Namespace, pod.Name, serverPodName, config.ServerReadyMessage, VolumeServerPodStartupTimeout) + _, err := e2etodopod.LookForStringInLog(pod.Namespace, pod.Name, serverPodName, config.ServerReadyMessage, VolumeServerPodStartupTimeout) framework.ExpectNoError(err, "Failed to find %q in pod logs: %s", config.ServerReadyMessage, err) } return pod @@ -475,7 +477,7 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string // Block: check content deviceName := fmt.Sprintf("/opt/%d", i) commands := GenerateReadBlockCmd(deviceName, len(test.ExpectedContent)) - _, err := framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute) + _, err := e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute) framework.ExpectNoError(err, "failed: finding the contents of the block device %s.", deviceName) // Check that it's a real block device @@ -484,7 +486,7 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string // Filesystem: check content fileName := fmt.Sprintf("/opt/%d/%s", i, test.File) commands := GenerateReadFileCmd(fileName) - _, err := framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute) + _, err := e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute) framework.ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName) // Check that a directory has been mounted @@ -495,14 +497,14 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string // Filesystem: check fsgroup if fsGroup != nil { ginkgo.By("Checking fsGroup is correct.") - _, err = framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"ls", "-ld", dirName}, strconv.Itoa(int(*fsGroup)), time.Minute) + _, err = e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"ls", "-ld", dirName}, strconv.Itoa(int(*fsGroup)), time.Minute) framework.ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup)) } // Filesystem: check fsType if fsType != "" { ginkgo.By("Checking fsType is correct.") - _, err = framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"grep", " " + dirName + " ", "/proc/mounts"}, fsType, time.Minute) + _, err = e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"grep", " " + dirName + " ", "/proc/mounts"}, fsType, time.Minute) framework.ExpectNoError(err, "failed: getting the right fsType %s", fsType) } } @@ -587,7 +589,7 @@ func InjectContent(f *framework.Framework, config TestConfig, fsGroup *int64, fs fileName := fmt.Sprintf("/opt/%d/%s", i, test.File) commands = append(commands, generateWriteFileCmd(test.ExpectedContent, fileName)...) } - out, err := framework.RunKubectl(injectorPod.Namespace, commands...) + out, err := e2etodokubectl.RunKubectl(injectorPod.Namespace, commands...) framework.ExpectNoError(err, "failed: writing the contents: %s", out) }