e2e framework: move kubectl and pod helper code

This commit is contained in:
Patrick Ohly 2022-08-25 12:45:36 +02:00
parent e95c2dbbe3
commit 2c8ef492ae
30 changed files with 675 additions and 589 deletions

View File

@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -79,7 +80,7 @@ var _ = SIGDescribe("ConfigMap", func() {
},
}
f.TestContainerOutput("consume configMaps", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{
"CONFIG_DATA_1=value-1",
})
})
@ -123,7 +124,7 @@ var _ = SIGDescribe("ConfigMap", func() {
},
}
f.TestContainerOutput("consume configMaps", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{
"data-1=value-1", "data-2=value-2", "data-3=value-3",
"p-data-1=value-1", "p-data-2=value-2", "p-data-3=value-3",
})

View File

@ -23,6 +23,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -57,7 +58,7 @@ var _ = SIGDescribe("Containers", func() {
*/
framework.ConformanceIt("should be able to override the image's default arguments (container cmd) [NodeConformance]", func() {
pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester", "override", "arguments")
f.TestContainerOutput("override arguments", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "override arguments", pod, 0, []string{
"[/agnhost entrypoint-tester override arguments]",
})
})
@ -73,7 +74,7 @@ var _ = SIGDescribe("Containers", func() {
pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester")
pod.Spec.Containers[0].Command = []string{"/agnhost-2"}
f.TestContainerOutput("override command", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "override command", pod, 0, []string{
"[/agnhost-2 entrypoint-tester]",
})
})
@ -87,7 +88,7 @@ var _ = SIGDescribe("Containers", func() {
pod := entrypointTestPod(f.Namespace.Name, "entrypoint-tester", "override", "arguments")
pod.Spec.Containers[0].Command = []string{"/agnhost-2"}
f.TestContainerOutput("override all", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "override all", pod, 0, []string{
"[/agnhost-2 entrypoint-tester override arguments]",
})
})

View File

@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -418,5 +419,5 @@ func testDownwardAPI(f *framework.Framework, podName string, env []v1.EnvVar, ex
}
func testDownwardAPIUsingPod(f *framework.Framework, pod *v1.Pod, env []v1.EnvVar, expectations []string) {
f.TestContainerOutputRegexp("downward api env vars", pod, 0, expectations)
e2etodopod.TestContainerOutputRegexp(f, "downward api env vars", pod, 0, expectations)
}

View File

@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -57,7 +58,7 @@ var _ = SIGDescribe("Variable Expansion", func() {
}
pod := newPod([]string{"sh", "-c", "env"}, envVars, nil, nil)
f.TestContainerOutput("env composition", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "env composition", pod, 0, []string{
"FOO=foo-value",
"BAR=bar-value",
"FOOBAR=foo-value;;bar-value",
@ -78,7 +79,7 @@ var _ = SIGDescribe("Variable Expansion", func() {
}
pod := newPod([]string{"sh", "-c", "TEST_VAR=wrong echo \"$(TEST_VAR)\""}, envVars, nil, nil)
f.TestContainerOutput("substitution in container's command", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "substitution in container's command", pod, 0, []string{
"test-value",
})
})
@ -98,7 +99,7 @@ var _ = SIGDescribe("Variable Expansion", func() {
pod := newPod([]string{"sh", "-c"}, envVars, nil, nil)
pod.Spec.Containers[0].Args = []string{"TEST_VAR=wrong echo \"$(TEST_VAR)\""}
f.TestContainerOutput("substitution in container's args", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "substitution in container's args", pod, 0, []string{
"test-value",
})
})
@ -138,7 +139,7 @@ var _ = SIGDescribe("Variable Expansion", func() {
envVars[0].Value = pod.ObjectMeta.Name
pod.Spec.Containers[0].Command = []string{"sh", "-c", "test -d /testcontainer/" + pod.ObjectMeta.Name + ";echo $?"}
f.TestContainerOutput("substitution in volume subpath", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "substitution in volume subpath", pod, 0, []string{
"0",
})
})

View File

@ -51,6 +51,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
e2ewebsocket "k8s.io/kubernetes/test/e2e/framework/websocket"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -522,7 +523,7 @@ var _ = SIGDescribe("Pods", func() {
"FOOSERVICE_PORT_8765_TCP_ADDR=",
}
expectNoErrorWithRetries(func() error {
return f.MatchContainerOutput(pod, containerName, expectedVars, gomega.ContainSubstring)
return e2etodopod.MatchContainerOutput(f, pod, containerName, expectedVars, gomega.ContainSubstring)
}, maxRetries, "Container should have service environment variables set")
})

View File

@ -29,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -81,7 +82,7 @@ var _ = SIGDescribe("Secrets", func() {
},
}
f.TestContainerOutput("consume secrets", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume secrets", pod, 0, []string{
"SECRET_DATA=value-1",
})
})
@ -125,7 +126,7 @@ var _ = SIGDescribe("Secrets", func() {
},
}
f.TestContainerOutput("consume secrets", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume secrets", pod, 0, []string{
"data-1=value-1", "data-2=value-2", "data-3=value-3",
"p-data-1=value-1", "p-data-2=value-2", "p-data-3=value-3",
})

View File

@ -29,6 +29,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
"k8s.io/utils/pointer"
@ -115,7 +116,7 @@ var _ = SIGDescribe("Security Context", func() {
// When running in the host's user namespace, the /proc/self/uid_map file content looks like:
// 0 0 4294967295
// Verify the value 4294967295 is present in the output.
f.TestContainerOutput("read namespace", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "read namespace", pod, 0, []string{
"4294967295",
})
})
@ -239,7 +240,7 @@ var _ = SIGDescribe("Security Context", func() {
// Each line should be "=0" that means root inside the container is the owner of the file.
downwardAPIVolFiles := 1
projectedFiles := len(secret.Data) + downwardAPIVolFiles
f.TestContainerOutput("check file permissions", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "check file permissions", pod, 0, []string{
strings.Repeat("=0\n", len(secret.Data)+len(configMap.Data)+downwardAPIVolFiles+projectedFiles),
})
})
@ -299,7 +300,7 @@ var _ = SIGDescribe("Security Context", func() {
// Expect one line for each file on all the volumes.
// Each line should be "=200" (fsGroup) that means it was mapped to the
// right user inside the container.
f.TestContainerOutput("check FSGroup is mapped correctly", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "check FSGroup is mapped correctly", pod, 0, []string{
strings.Repeat(fmt.Sprintf("=%v\n", fsGroup), len(configMap.Data)),
})
})

View File

@ -30,6 +30,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -485,7 +486,7 @@ var _ = SIGDescribe("ConfigMap", func() {
},
}
f.TestContainerOutput("consume configMaps", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{
"content of file \"/etc/configmap-volume/data-1\": value-1",
})
@ -621,7 +622,7 @@ func doConfigMapE2EWithoutMappings(f *framework.Framework, asUser bool, fsGroup
"content of file \"/etc/configmap-volume/data-1\": value-1",
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume configMaps", pod, 0, output)
e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output)
}
func doConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int64, itemMode *int32) {
@ -673,7 +674,7 @@ func doConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int
fileModeRegexp := getFileModeRegex("/etc/configmap-volume/path/to/data-2", itemMode)
output = append(output, fileModeRegexp)
}
f.TestContainerOutputRegexp("consume configMaps", pod, 0, output)
e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output)
}
func createNonOptionalConfigMapPod(f *framework.Framework, volumeMountPath string) (*v1.Pod, error) {

View File

@ -24,6 +24,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -133,5 +134,5 @@ func testDownwardAPIForEphemeralStorage(f *framework.Framework, podName string,
}
func testDownwardAPIUsingPod(f *framework.Framework, pod *v1.Pod, env []v1.EnvVar, expectations []string) {
f.TestContainerOutputRegexp("downward api env vars", pod, 0, expectations)
e2etodopod.TestContainerOutputRegexp(f, "downward api env vars", pod, 0, expectations)
}

View File

@ -27,6 +27,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -53,7 +54,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("%s\n", podName),
})
})
@ -69,7 +70,7 @@ var _ = SIGDescribe("Downward API volume", func() {
defaultMode := int32(0400)
pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--------",
})
})
@ -85,7 +86,7 @@ var _ = SIGDescribe("Downward API volume", func() {
mode := int32(0400)
pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--------",
})
})
@ -100,7 +101,7 @@ var _ = SIGDescribe("Downward API volume", func() {
FSGroup: &gid,
}
setPodNonRootUser(pod)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("%s\n", podName),
})
})
@ -116,7 +117,7 @@ var _ = SIGDescribe("Downward API volume", func() {
FSGroup: &gid,
}
setPodNonRootUser(pod)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--r-----",
})
})
@ -193,7 +194,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("2\n"),
})
})
@ -207,7 +208,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("67108864\n"),
})
})
@ -221,7 +222,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("1\n"),
})
})
@ -235,7 +236,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("33554432\n"),
})
})
@ -249,7 +250,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit")
f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"})
e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"})
})
/*
@ -261,7 +262,7 @@ var _ = SIGDescribe("Downward API volume", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit")
f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"})
e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"})
})
})

View File

@ -29,6 +29,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -390,7 +391,7 @@ func doTestSetgidFSGroup(f *framework.Framework, uid int64, medium v1.StorageMed
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTestSubPathFSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -423,7 +424,7 @@ func doTestSubPathFSGroup(f *framework.Framework, uid int64, medium v1.StorageMe
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTestVolumeModeFSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -448,7 +449,7 @@ func doTestVolumeModeFSGroup(f *framework.Framework, uid int64, medium v1.Storag
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTest0644FSGroup(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -476,7 +477,7 @@ func doTest0644FSGroup(f *framework.Framework, uid int64, medium v1.StorageMediu
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTestVolumeMode(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -498,7 +499,7 @@ func doTestVolumeMode(f *framework.Framework, uid int64, medium v1.StorageMedium
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTest0644(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -523,7 +524,7 @@ func doTest0644(f *framework.Framework, uid int64, medium v1.StorageMedium) {
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTest0666(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -548,7 +549,7 @@ func doTest0666(f *framework.Framework, uid int64, medium v1.StorageMedium) {
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func doTest0777(f *framework.Framework, uid int64, medium v1.StorageMedium) {
@ -573,7 +574,7 @@ func doTest0777(f *framework.Framework, uid int64, medium v1.StorageMedium) {
if medium == v1.StorageMediumMemory {
out = append(out, "mount type of \"/test-volume\": tmpfs")
}
f.TestContainerOutput(msg, pod, 0, out)
e2etodopod.TestContainerOutput(f, msg, pod, 0, out)
}
func formatMedium(medium v1.StorageMedium) string {

View File

@ -21,9 +21,10 @@ import (
"os"
"path"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -58,7 +59,7 @@ var _ = SIGDescribe("HostPath", func() {
fmt.Sprintf("--fs_type=%v", volumePath),
fmt.Sprintf("--file_mode=%v", volumePath),
}
f.TestContainerOutputRegexp("hostPath mode", pod, 0, []string{
e2etodopod.TestContainerOutputRegexp(f, "hostPath mode", pod, 0, []string{
"mode of file \"/test-volume\": dg?trwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir
})
})
@ -87,7 +88,7 @@ var _ = SIGDescribe("HostPath", func() {
}
//Read the content of the file with the second container to
//verify volumes being shared properly among containers within the pod.
f.TestContainerOutput("hostPath r/w", pod, 1, []string{
e2etodopod.TestContainerOutput(f, "hostPath r/w", pod, 1, []string{
"content of file \"/test-volume/test-file\": mount-tester new file",
})
})
@ -124,7 +125,7 @@ var _ = SIGDescribe("HostPath", func() {
fmt.Sprintf("--retry_time=%d", retryDuration),
}
f.TestContainerOutput("hostPath subPath", pod, 1, []string{
e2etodopod.TestContainerOutput(f, "hostPath subPath", pod, 1, []string{
"content of file \"" + filePathInReader + "\": mount-tester new file",
})
})

View File

@ -24,6 +24,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -88,7 +89,7 @@ var _ = SIGDescribe("Projected combined", func() {
},
},
}
f.TestContainerOutput("Check all projections for projected volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "Check all projections for projected volume plugin", pod, 0, []string{
podName,
"secret-value-1",
"configmap-value-1",

View File

@ -27,6 +27,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -450,7 +451,7 @@ var _ = SIGDescribe("Projected configMap", func() {
},
}
f.TestContainerOutput("consume configMaps", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "consume configMaps", pod, 0, []string{
"content of file \"/etc/projected-configmap-volume/data-1\": value-1",
})
@ -512,7 +513,7 @@ func doProjectedConfigMapE2EWithoutMappings(f *framework.Framework, asUser bool,
"content of file \"/etc/projected-configmap-volume/data-1\": value-1",
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume configMaps", pod, 0, output)
e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output)
}
func doProjectedConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fsGroup int64, itemMode *int32) {
@ -563,7 +564,7 @@ func doProjectedConfigMapE2EWithMappings(f *framework.Framework, asUser bool, fs
fileModeRegexp := getFileModeRegex("/etc/projected-configmap-volume/path/to/data-2", itemMode)
output = append(output, fileModeRegexp)
}
f.TestContainerOutputRegexp("consume configMaps", pod, 0, output)
e2etodopod.TestContainerOutputRegexp(f, "consume configMaps", pod, 0, output)
}
func createProjectedConfigMapMounttestPod(namespace, volumeName, referenceName, mountPath string, mounttestArgs ...string) *v1.Pod {

View File

@ -26,6 +26,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -53,7 +54,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("%s\n", podName),
})
})
@ -69,7 +70,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
defaultMode := int32(0400)
pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--------",
})
})
@ -85,7 +86,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
mode := int32(0400)
pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--------",
})
})
@ -100,7 +101,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
FSGroup: &gid,
}
setPodNonRootUser(pod)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("%s\n", podName),
})
})
@ -116,7 +117,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
FSGroup: &gid,
}
setPodNonRootUser(pod)
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
"mode of file \"/etc/podinfo/podname\": -r--r-----",
})
})
@ -193,7 +194,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("2\n"),
})
})
@ -207,7 +208,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("67108864\n"),
})
})
@ -221,7 +222,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("1\n"),
})
})
@ -235,7 +236,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request")
f.TestContainerOutput("downward API volume plugin", pod, 0, []string{
e2etodopod.TestContainerOutput(f, "downward API volume plugin", pod, 0, []string{
fmt.Sprintf("33554432\n"),
})
})
@ -249,7 +250,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit")
f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"})
e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"})
})
/*
@ -261,7 +262,7 @@ var _ = SIGDescribe("Projected downwardAPI", func() {
podName := "downwardapi-volume-" + string(uuid.NewUUID())
pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit")
f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"})
e2etodopod.TestContainerOutputRegexp(f, "downward API volume plugin", pod, 0, []string{"[1-9]"})
})
})

View File

@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -200,7 +201,7 @@ var _ = SIGDescribe("Projected secret", func() {
}
fileModeRegexp := getFileModeRegex("/etc/projected-secret-volume/data-1", nil)
f.TestContainerOutputRegexp("consume secrets", pod, 0, []string{
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, []string{
"content of file \"/etc/projected-secret-volume/data-1\": value-1",
fileModeRegexp,
})
@ -504,7 +505,7 @@ func doProjectedSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput)
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput)
}
func doProjectedSecretE2EWithMapping(f *framework.Framework, mode *int32) {
@ -581,5 +582,5 @@ func doProjectedSecretE2EWithMapping(f *framework.Framework, mode *int32) {
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput)
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput)
}

View File

@ -27,6 +27,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -190,7 +191,7 @@ var _ = SIGDescribe("Secrets", func() {
}
fileModeRegexp := getFileModeRegex("/etc/secret-volume/data-1", nil)
f.TestContainerOutputRegexp("consume secrets", pod, 0, []string{
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, []string{
"content of file \"/etc/secret-volume/data-1\": value-1",
fileModeRegexp,
})
@ -534,7 +535,7 @@ func doSecretE2EWithoutMapping(f *framework.Framework, defaultMode *int32, secre
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput)
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput)
}
func doSecretE2EWithMapping(f *framework.Framework, mode *int32) {
@ -602,7 +603,7 @@ func doSecretE2EWithMapping(f *framework.Framework, mode *int32) {
fileModeRegexp,
}
f.TestContainerOutputRegexp("consume secrets", pod, 0, expectedOutput)
e2etodopod.TestContainerOutputRegexp(f, "consume secrets", pod, 0, expectedOutput)
}
func createNonOptionalSecretPod(f *framework.Framework, volumeMountPath, podName string) error {

View File

@ -50,7 +50,6 @@ import (
admissionapi "k8s.io/pod-security-admission/api"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
// TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245)
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
@ -563,20 +562,6 @@ func (f *Framework) ClientConfig() *rest.Config {
return ret
}
// TestContainerOutput runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using a substring matcher.
func (f *Framework) TestContainerOutput(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, gomega.ContainSubstring)
}
// TestContainerOutputRegexp runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using a regexp matcher.
func (f *Framework) TestContainerOutputRegexp(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, gomega.MatchRegexp)
}
// KubeUser is a struct for managing kubernetes user info.
type KubeUser struct {
Name string `yaml:"name"`

View File

@ -61,6 +61,7 @@ import (
e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl"
testutils "k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
@ -465,10 +466,10 @@ func (j *TestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[stri
}
j.Logger.Infof("creating replication controller")
framework.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-")
e2etodokubectl.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-")
j.Logger.Infof("creating service")
framework.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-")
e2etodokubectl.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-")
if len(svcAnnotations) > 0 {
svcList, err := j.Client.CoreV1().Services(ns).List(context.TODO(), metav1.ListOptions{})
framework.ExpectNoError(err)
@ -481,7 +482,7 @@ func (j *TestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[stri
if exists("secret.yaml") {
j.Logger.Infof("creating secret")
framework.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-")
e2etodokubectl.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-")
}
j.Logger.Infof("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
@ -550,7 +551,7 @@ func (j *TestJig) runCreate(ing *networkingv1.Ingress) (*networkingv1.Ingress, e
if err := ingressToManifest(ing, filePath); err != nil {
return nil, err
}
_, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
_, err := e2etodokubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
return ing, err
}
@ -565,14 +566,14 @@ func (j *TestJig) runUpdate(ing *networkingv1.Ingress) (*networkingv1.Ingress, e
if err := ingressToManifest(ing, filePath); err != nil {
return nil, err
}
_, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
_, err := e2etodokubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
return ing, err
}
// DescribeIng describes information of ingress by running kubectl describe ing.
func DescribeIng(ns string) {
framework.Logf("\nOutput of kubectl describe ing:\n")
desc, _ := framework.RunKubectl(
desc, _ := e2etodokubectl.RunKubectl(
ns, "describe", "ing")
framework.Logf(desc)
}
@ -680,7 +681,7 @@ func (j *TestJig) runDelete(ing *networkingv1.Ingress) error {
if err := ingressToManifest(ing, filePath); err != nil {
return err
}
_, err := framework.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
_, err := e2etodokubectl.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
return err
}
@ -688,7 +689,7 @@ func (j *TestJig) runDelete(ing *networkingv1.Ingress) error {
// TODO(nikhiljindal): Update this to be able to return hostname as well.
func getIngressAddressFromKubemci(name string) ([]string, error) {
var addresses []string
out, err := framework.RunKubemciCmd("get-status", name)
out, err := e2etodokubectl.RunKubemciCmd("get-status", name)
if err != nil {
return addresses, err
}
@ -1032,7 +1033,7 @@ func (cont *NginxIngressController) Init() {
}
framework.Logf("initializing nginx ingress controller")
framework.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-")
e2etodokubectl.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-")
rc, err := cont.Client.CoreV1().ReplicationControllers(cont.Ns).Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
framework.ExpectNoError(err)

View File

@ -45,6 +45,8 @@ import (
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
netutils "k8s.io/utils/net"
)
@ -248,7 +250,7 @@ func (config *NetworkingTestConfig) diagnoseMissingEndpoints(foundEndpoints sets
continue
}
framework.Logf("\nOutput of kubectl describe pod %v/%v:\n", e.Namespace, e.Name)
desc, _ := framework.RunKubectl(
desc, _ := e2etodokubectl.RunKubectl(
e.Namespace, "describe", "pod", e.Name, fmt.Sprintf("--namespace=%v", e.Namespace))
framework.Logf(desc)
}
@ -519,7 +521,7 @@ func (config *NetworkingTestConfig) executeCurlCmd(cmd string, expected string)
podName := config.HostTestContainerPod.Name
var msg string
if pollErr := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) {
stdout, err := framework.RunHostCmd(config.Namespace, podName, cmd)
stdout, err := e2etodopod.RunHostCmd(config.Namespace, podName, cmd)
if err != nil {
msg = fmt.Sprintf("failed executing cmd %v in %v/%v: %v", cmd, config.Namespace, podName, err)
framework.Logf(msg)
@ -533,7 +535,7 @@ func (config *NetworkingTestConfig) executeCurlCmd(cmd string, expected string)
return true, nil
}); pollErr != nil {
framework.Logf("\nOutput of kubectl describe pod %v/%v:\n", config.Namespace, podName)
desc, _ := framework.RunKubectl(
desc, _ := e2etodokubectl.RunKubectl(
config.Namespace, "describe", "pod", podName, fmt.Sprintf("--namespace=%v", config.Namespace))
framework.Logf("%s", desc)
framework.Failf("Timed out in %v: %v", retryTimeout, msg)

View File

@ -17,9 +17,6 @@ limitations under the License.
package framework
import (
"fmt"
"os"
"path"
"sync"
"time"
@ -34,38 +31,6 @@ import (
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
)
const etcdImage = "3.5.5-0"
// EtcdUpgrade upgrades etcd on GCE.
func EtcdUpgrade(targetStorage, targetVersion string) error {
switch TestContext.Provider {
case "gce":
return etcdUpgradeGCE(targetStorage, targetVersion)
default:
return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
}
}
func etcdUpgradeGCE(targetStorage, targetVersion string) error {
env := append(
os.Environ(),
"TEST_ETCD_VERSION="+targetVersion,
"STORAGE_BACKEND="+targetStorage,
"TEST_ETCD_IMAGE="+etcdImage)
_, _, err := RunCmdEnv(env, GCEUpgradeScript(), "-l", "-M")
return err
}
// LocationParamGKE returns parameter related to location for gcloud command.
func LocationParamGKE() string {
if TestContext.CloudConfig.MultiMaster {
// GKE Regional Clusters are being tested.
return fmt.Sprintf("--region=%s", TestContext.CloudConfig.Region)
}
return fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone)
}
// AppendContainerCommandGroupIfNeeded returns container command group parameter if necessary.
func AppendContainerCommandGroupIfNeeded(args []string) []string {
if TestContext.CloudConfig.Region != "" {
@ -75,55 +40,6 @@ func AppendContainerCommandGroupIfNeeded(args []string) []string {
return args
}
// MasterUpgradeGKE upgrades master node to the specified version on GKE.
func MasterUpgradeGKE(namespace string, v string) error {
Logf("Upgrading master to %q", v)
args := []string{
"container",
"clusters",
fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
LocationParamGKE(),
"upgrade",
TestContext.CloudConfig.Cluster,
"--master",
fmt.Sprintf("--cluster-version=%s", v),
"--quiet",
}
_, _, err := RunCmd("gcloud", AppendContainerCommandGroupIfNeeded(args)...)
if err != nil {
return err
}
WaitForSSHTunnels(namespace)
return nil
}
// GCEUpgradeScript returns path of script for upgrading on GCE.
func GCEUpgradeScript() string {
if len(TestContext.GCEUpgradeScript) == 0 {
return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
}
return TestContext.GCEUpgradeScript
}
// WaitForSSHTunnels waits for establishing SSH tunnel to busybox pod.
func WaitForSSHTunnels(namespace string) {
Logf("Waiting for SSH tunnels to establish")
RunKubectl(namespace, "run", "ssh-tunnel-test",
"--image=busybox",
"--restart=Never",
"--command", "--",
"echo", "Hello")
defer RunKubectl(namespace, "delete", "pod", "ssh-tunnel-test")
// allow up to a minute for new ssh tunnels to establish
wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
_, err := RunKubectl(namespace, "logs", "ssh-tunnel-test")
return err == nil, nil
})
}
// NodeKiller is a utility to simulate node failures.
type NodeKiller struct {
config NodeKillerConfig

View File

@ -46,6 +46,7 @@ import (
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
testutils "k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
netutils "k8s.io/utils/net"
@ -910,7 +911,7 @@ func testEndpointReachability(endpoint string, port int32, protocol v1.Protocol,
}
err := wait.PollImmediate(1*time.Second, ServiceReachabilityShortPollTimeout, func() (bool, error) {
_, err := framework.RunHostCmd(execPod.Namespace, execPod.Name, cmd)
_, err := e2etodopod.RunHostCmd(execPod.Namespace, execPod.Name, cmd)
if err != nil {
framework.Logf("Service reachability failing with error: %v\nRetrying...", err)
return false, nil
@ -1002,7 +1003,7 @@ func (j *TestJig) checkExternalServiceReachability(svc *v1.Service, pod *v1.Pod)
// Service must resolve to IP
cmd := fmt.Sprintf("nslookup %s", svcName)
return wait.PollImmediate(framework.Poll, ServiceReachabilityShortPollTimeout, func() (done bool, err error) {
_, stderr, err := framework.RunHostCmdWithFullOutput(pod.Namespace, pod.Name, cmd)
_, stderr, err := e2etodopod.RunHostCmdWithFullOutput(pod.Namespace, pod.Name, cmd)
// NOTE(claudiub): nslookup may return 0 on Windows, even though the DNS name was not found. In this case,
// we can check stderr for the error.
if err != nil || (framework.NodeOSDistroIs("windows") && strings.Contains(stderr, fmt.Sprintf("can't find %s", svcName))) {

View File

@ -30,6 +30,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubectl/pkg/util/podutils"
"k8s.io/kubernetes/test/e2e/framework"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
)
@ -165,7 +166,7 @@ func ResumeNextPod(c clientset.Interface, ss *appsv1.StatefulSet) {
if resumedPod != "" {
framework.Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod)
}
_, err := framework.RunHostCmdWithRetries(pod.Namespace, pod.Name, "dd if=/dev/zero of=/data/statefulset-continue bs=1 count=1 conv=fsync", StatefulSetPoll, StatefulPodTimeout)
_, err := e2etodopod.RunHostCmdWithRetries(pod.Namespace, pod.Name, "dd if=/dev/zero of=/data/statefulset-continue bs=1 count=1 conv=fsync", StatefulSetPoll, StatefulPodTimeout)
framework.ExpectNoError(err)
framework.Logf("Resumed pod %v", pod.Name)
resumedPod = pod.Name

View File

@ -34,6 +34,7 @@ import (
"k8s.io/kubectl/pkg/util/podutils"
"k8s.io/kubernetes/test/e2e/framework"
e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
)
// CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the Namespace ns using kubectl create.
@ -192,7 +193,7 @@ func CheckHostname(c clientset.Interface, ss *appsv1.StatefulSet) error {
cmd := "printf $(hostname)"
podList := GetPodList(c, ss)
for _, statefulPod := range podList.Items {
hostname, err := framework.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
hostname, err := e2etodopod.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
if err != nil {
return err
}
@ -236,7 +237,7 @@ func CheckServiceName(ss *appsv1.StatefulSet, expectedServiceName string) error
func ExecInStatefulPods(c clientset.Interface, ss *appsv1.StatefulSet, cmd string) error {
podList := GetPodList(c, ss)
for _, statefulPod := range podList.Items {
stdout, err := framework.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
stdout, err := e2etodopod.RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
framework.Logf("stdout of %v on %v: %v", cmd, statefulPod.Name, stdout)
if err != nil {
return err

View File

@ -0,0 +1,192 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pod
import (
"bytes"
"fmt"
"io"
"net"
"net/url"
"os/exec"
"strings"
"syscall"
"time"
"k8s.io/client-go/tools/clientcmd"
uexec "k8s.io/utils/exec"
"k8s.io/kubernetes/test/e2e/framework"
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
)
// KubectlBuilder is used to build, customize and execute a kubectl Command.
// Add more functions to customize the builder as needed.
type KubectlBuilder struct {
cmd *exec.Cmd
timeout <-chan time.Time
}
// NewKubectlCommand returns a KubectlBuilder for running kubectl.
func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder {
b := new(KubectlBuilder)
tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, namespace)
b.cmd = tk.KubectlCmd(args...)
return b
}
// WithEnv sets the given environment and returns itself.
func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder {
b.cmd.Env = env
return b
}
// WithTimeout sets the given timeout and returns itself.
func (b *KubectlBuilder) WithTimeout(t <-chan time.Time) *KubectlBuilder {
b.timeout = t
return b
}
// WithStdinData sets the given data to stdin and returns itself.
func (b KubectlBuilder) WithStdinData(data string) *KubectlBuilder {
b.cmd.Stdin = strings.NewReader(data)
return &b
}
// WithStdinReader sets the given reader and returns itself.
func (b KubectlBuilder) WithStdinReader(reader io.Reader) *KubectlBuilder {
b.cmd.Stdin = reader
return &b
}
// ExecOrDie runs the kubectl executable or dies if error occurs.
func (b KubectlBuilder) ExecOrDie(namespace string) string {
str, err := b.Exec()
// In case of i/o timeout error, try talking to the apiserver again after 2s before dying.
// Note that we're still dying after retrying so that we can get visibility to triage it further.
if isTimeout(err) {
framework.Logf("Hit i/o timeout error, talking to the server 2s later to see if it's temporary.")
time.Sleep(2 * time.Second)
retryStr, retryErr := RunKubectl(namespace, "version")
framework.Logf("stdout: %q", retryStr)
framework.Logf("err: %v", retryErr)
}
framework.ExpectNoError(err)
return str
}
func isTimeout(err error) bool {
switch err := err.(type) {
case *url.Error:
if err, ok := err.Err.(net.Error); ok && err.Timeout() {
return true
}
case net.Error:
if err.Timeout() {
return true
}
}
return false
}
// Exec runs the kubectl executable.
func (b KubectlBuilder) Exec() (string, error) {
stdout, _, err := b.ExecWithFullOutput()
return stdout, err
}
// ExecWithFullOutput runs the kubectl executable, and returns the stdout and stderr.
func (b KubectlBuilder) ExecWithFullOutput() (string, string, error) {
var stdout, stderr bytes.Buffer
cmd := b.cmd
cmd.Stdout, cmd.Stderr = &stdout, &stderr
framework.Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
if err := cmd.Start(); err != nil {
return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err)
}
errCh := make(chan error, 1)
go func() {
errCh <- cmd.Wait()
}()
select {
case err := <-errCh:
if err != nil {
var rc = 127
if ee, ok := err.(*exec.ExitError); ok {
rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus())
framework.Logf("rc: %d", rc)
}
return stdout.String(), stderr.String(), uexec.CodeExitError{
Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err),
Code: rc,
}
}
case <-b.timeout:
b.cmd.Process.Kill()
return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr)
}
framework.Logf("stderr: %q", stderr.String())
framework.Logf("stdout: %q", stdout.String())
return stdout.String(), stderr.String(), nil
}
// RunKubectlOrDie is a convenience wrapper over kubectlBuilder
func RunKubectlOrDie(namespace string, args ...string) string {
return NewKubectlCommand(namespace, args...).ExecOrDie(namespace)
}
// RunKubectl is a convenience wrapper over kubectlBuilder
func RunKubectl(namespace string, args ...string) (string, error) {
return NewKubectlCommand(namespace, args...).Exec()
}
// RunKubectlWithFullOutput is a convenience wrapper over kubectlBuilder
// It will also return the command's stderr.
func RunKubectlWithFullOutput(namespace string, args ...string) (string, string, error) {
return NewKubectlCommand(namespace, args...).ExecWithFullOutput()
}
// RunKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func RunKubectlOrDieInput(namespace string, data string, args ...string) string {
return NewKubectlCommand(namespace, args...).WithStdinData(data).ExecOrDie(namespace)
}
// RunKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func RunKubectlInput(namespace string, data string, args ...string) (string, error) {
return NewKubectlCommand(namespace, args...).WithStdinData(data).Exec()
}
// RunKubemciWithKubeconfig is a convenience wrapper over RunKubemciCmd
func RunKubemciWithKubeconfig(args ...string) (string, error) {
if framework.TestContext.KubeConfig != "" {
args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+framework.TestContext.KubeConfig)
}
return RunKubemciCmd(args...)
}
// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
// It assumes that kubemci exists in PATH.
func RunKubemciCmd(args ...string) (string, error) {
// kubemci is assumed to be in PATH.
kubemci := "kubemci"
b := new(KubectlBuilder)
args = append(args, "--gcp-project="+framework.TestContext.CloudConfig.ProjectID)
b.cmd = exec.Command(kubemci, args...)
return b.Exec()
}

View File

@ -0,0 +1,42 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"time"
"k8s.io/apimachinery/pkg/util/wait"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl"
)
// WaitForSSHTunnels waits for establishing SSH tunnel to busybox pod.
func WaitForSSHTunnels(namespace string) {
e2elog.Logf("Waiting for SSH tunnels to establish")
e2etodokubectl.RunKubectl(namespace, "run", "ssh-tunnel-test",
"--image=busybox",
"--restart=Never",
"--command", "--",
"echo", "Hello")
defer e2etodokubectl.RunKubectl(namespace, "delete", "pod", "ssh-tunnel-test")
// allow up to a minute for new ssh tunnels to establish
wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
_, err := e2etodokubectl.RunKubectl(namespace, "logs", "ssh-tunnel-test")
return err == nil, nil
})
}

View File

@ -0,0 +1,238 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pod
import (
"context"
"fmt"
"strings"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
gomegatypes "github.com/onsi/gomega/types"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
clientset "k8s.io/client-go/kubernetes"
apiv1pod "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl"
)
// DEPRECATED constants. Use the timeouts in framework.Framework instead.
const (
// Poll is how often to Poll pods, nodes and claims.
Poll = 2 * time.Second
)
// LookForStringInPodExec looks for the given string in the output of a command
// executed in the first container of specified pod.
func LookForStringInPodExec(ns, podName string, command []string, expectedString string, timeout time.Duration) (result string, err error) {
return LookForStringInPodExecToContainer(ns, podName, "", command, expectedString, timeout)
}
// LookForStringInPodExecToContainer looks for the given string in the output of a
// command executed in specified pod container, or first container if not specified.
func LookForStringInPodExecToContainer(ns, podName, containerName string, command []string, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns)}
if len(containerName) > 0 {
args = append(args, fmt.Sprintf("--container=%s", containerName))
}
args = append(args, "--")
args = append(args, command...)
return e2etodokubectl.RunKubectlOrDie(ns, args...)
})
}
// lookForString looks for the given string in the output of fn, repeatedly calling fn until
// the timeout is reached or the string is found. Returns last log and possibly
// error if the string was not found.
func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) {
for t := time.Now(); time.Since(t) < timeout; time.Sleep(Poll) {
result = fn()
if strings.Contains(result, expectedString) {
return
}
}
err = fmt.Errorf("Failed to find \"%s\", last result: \"%s\"", expectedString, result)
return
}
// RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec`
// inside of a shell.
func RunHostCmd(ns, name, cmd string) (string, error) {
return e2etodokubectl.RunKubectl(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd)
}
// RunHostCmdWithFullOutput runs the given cmd in the context of the given pod using `kubectl exec`
// inside of a shell. It will also return the command's stderr.
func RunHostCmdWithFullOutput(ns, name, cmd string) (string, string, error) {
return e2etodokubectl.RunKubectlWithFullOutput(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd)
}
// RunHostCmdOrDie calls RunHostCmd and dies on error.
func RunHostCmdOrDie(ns, name, cmd string) string {
stdout, err := RunHostCmd(ns, name, cmd)
framework.Logf("stdout: %v", stdout)
framework.ExpectNoError(err)
return stdout
}
// RunHostCmdWithRetries calls RunHostCmd and retries all errors
// until it succeeds or the specified timeout expires.
// This can be used with idempotent commands to deflake transient Node issues.
func RunHostCmdWithRetries(ns, name, cmd string, interval, timeout time.Duration) (string, error) {
start := time.Now()
for {
out, err := RunHostCmd(ns, name, cmd)
if err == nil {
return out, nil
}
if elapsed := time.Since(start); elapsed > timeout {
return out, fmt.Errorf("RunHostCmd still failed after %v: %v", elapsed, err)
}
framework.Logf("Waiting %v to retry failed RunHostCmd: %v", interval, err)
time.Sleep(interval)
}
}
// LookForStringInLog looks for the given string in the log of a specific pod container
func LookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return e2etodokubectl.RunKubectlOrDie(ns, "logs", podName, container)
})
}
// CreateEmptyFileOnPod creates empty file at given path on the pod.
func CreateEmptyFileOnPod(namespace string, podName string, filePath string) error {
_, err := e2etodokubectl.RunKubectl(namespace, "exec", podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath))
return err
}
// DumpDebugInfo dumps debug info of tests.
func DumpDebugInfo(c clientset.Interface, ns string) {
sl, _ := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.Everything().String()})
for _, s := range sl.Items {
desc, _ := e2etodokubectl.RunKubectl(ns, "describe", "po", s.Name)
framework.Logf("\nOutput of kubectl describe %v:\n%v", s.Name, desc)
l, _ := e2etodokubectl.RunKubectl(ns, "logs", s.Name, "--tail=100")
framework.Logf("\nLast 100 log lines of %v:\n%v", s.Name, l)
}
}
// MatchContainerOutput creates a pod and waits for all it's containers to exit with success.
// It then tests that the matcher with each expectedOutput matches the output of the specified container.
func MatchContainerOutput(
f *framework.Framework,
pod *v1.Pod,
containerName string,
expectedOutput []string,
matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error {
ns := pod.ObjectMeta.Namespace
if ns == "" {
ns = f.Namespace.Name
}
podClient := f.PodClientNS(ns)
createdPod := podClient.Create(pod)
defer func() {
ginkgo.By("delete the pod")
podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
}()
// Wait for client pod to complete.
podErr := e2epod.WaitForPodSuccessInNamespaceTimeout(f.ClientSet, createdPod.Name, ns, f.Timeouts.PodStart)
// Grab its logs. Get host first.
podStatus, err := podClient.Get(context.TODO(), createdPod.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get pod status: %v", err)
}
if podErr != nil {
// Pod failed. Dump all logs from all containers to see what's wrong
_ = apiv1pod.VisitContainers(&podStatus.Spec, apiv1pod.AllFeatureEnabledContainers(), func(c *v1.Container, containerType apiv1pod.ContainerType) bool {
logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, c.Name)
if err != nil {
framework.Logf("Failed to get logs from node %q pod %q container %q: %v",
podStatus.Spec.NodeName, podStatus.Name, c.Name, err)
} else {
framework.Logf("Output of node %q pod %q container %q: %s", podStatus.Spec.NodeName, podStatus.Name, c.Name, logs)
}
return true
})
return fmt.Errorf("expected pod %q success: %v", createdPod.Name, podErr)
}
framework.Logf("Trying to get logs from node %s pod %s container %s: %v",
podStatus.Spec.NodeName, podStatus.Name, containerName, err)
// Sometimes the actual containers take a second to get started, try to get logs for 60s
logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, containerName)
if err != nil {
framework.Logf("Failed to get logs from node %q pod %q container %q. %v",
podStatus.Spec.NodeName, podStatus.Name, containerName, err)
return fmt.Errorf("failed to get logs from %s for %s: %v", podStatus.Name, containerName, err)
}
for _, expected := range expectedOutput {
m := matcher(expected)
matches, err := m.Match(logs)
if err != nil {
return fmt.Errorf("expected %q in container output: %v", expected, err)
} else if !matches {
return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs))
}
}
return nil
}
// TestContainerOutput runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using a substring matcher.
func TestContainerOutput(f *framework.Framework, scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
TestContainerOutputMatcher(f, scenarioName, pod, containerIndex, expectedOutput, gomega.ContainSubstring)
}
// TestContainerOutputRegexp runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using a regexp matcher.
func TestContainerOutputRegexp(f *framework.Framework, scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
TestContainerOutputMatcher(f, scenarioName, pod, containerIndex, expectedOutput, gomega.MatchRegexp)
}
// TestContainerOutputMatcher runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using the given matcher.
func TestContainerOutputMatcher(f *framework.Framework,
scenarioName string,
pod *v1.Pod,
containerIndex int,
expectedOutput []string,
matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) {
ginkgo.By(fmt.Sprintf("Creating a pod to test %v", scenarioName))
if containerIndex < 0 || containerIndex >= len(pod.Spec.Containers) {
framework.Failf("Invalid container index: %d", containerIndex)
}
framework.ExpectNoError(MatchContainerOutput(f, pod, pod.Spec.Containers[containerIndex].Name, expectedOutput, matcher))
}

View File

@ -0,0 +1,90 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package providers
import (
"fmt"
"os"
"path"
"k8s.io/kubernetes/test/e2e/framework"
e2etodonode "k8s.io/kubernetes/test/e2e/framework/todo/node"
)
const etcdImage = "3.5.5-0"
// EtcdUpgrade upgrades etcd on GCE.
func EtcdUpgrade(targetStorage, targetVersion string) error {
switch framework.TestContext.Provider {
case "gce":
return etcdUpgradeGCE(targetStorage, targetVersion)
default:
return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", framework.TestContext.Provider)
}
}
func etcdUpgradeGCE(targetStorage, targetVersion string) error {
env := append(
os.Environ(),
"TEST_ETCD_VERSION="+targetVersion,
"STORAGE_BACKEND="+targetStorage,
"TEST_ETCD_IMAGE="+etcdImage)
_, _, err := framework.RunCmdEnv(env, GCEUpgradeScript(), "-l", "-M")
return err
}
// LocationParamGKE returns parameter related to location for gcloud command.
func LocationParamGKE() string {
if framework.TestContext.CloudConfig.MultiMaster {
// GKE Regional Clusters are being tested.
return fmt.Sprintf("--region=%s", framework.TestContext.CloudConfig.Region)
}
return fmt.Sprintf("--zone=%s", framework.TestContext.CloudConfig.Zone)
}
// MasterUpgradeGKE upgrades master node to the specified version on GKE.
func MasterUpgradeGKE(namespace string, v string) error {
framework.Logf("Upgrading master to %q", v)
args := []string{
"container",
"clusters",
fmt.Sprintf("--project=%s", framework.TestContext.CloudConfig.ProjectID),
LocationParamGKE(),
"upgrade",
framework.TestContext.CloudConfig.Cluster,
"--master",
fmt.Sprintf("--cluster-version=%s", v),
"--quiet",
}
_, _, err := framework.RunCmd("gcloud", framework.AppendContainerCommandGroupIfNeeded(args)...)
if err != nil {
return err
}
e2etodonode.WaitForSSHTunnels(namespace)
return nil
}
// GCEUpgradeScript returns path of script for upgrading on GCE.
func GCEUpgradeScript() string {
if len(framework.TestContext.GCEUpgradeScript) == 0 {
return path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh")
}
return framework.TestContext.GCEUpgradeScript
}

View File

@ -23,7 +23,6 @@ import (
"fmt"
"io"
"math/rand"
"net"
"net/url"
"os"
"os/exec"
@ -32,18 +31,15 @@ import (
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
gomegatypes "github.com/onsi/gomega/types"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
@ -59,11 +55,10 @@ import (
watchtools "k8s.io/client-go/tools/watch"
testutils "k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
uexec "k8s.io/utils/exec"
netutils "k8s.io/utils/net"
// TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245)
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -75,10 +70,6 @@ const (
// TODO(justinsb): Avoid hardcoding this.
awsMasterIP = "172.20.0.9"
// AllContainers specifies that all containers be visited
// Copied from pkg/api/v1/pod to avoid pulling extra dependencies
AllContainers = InitContainers | Containers | EphemeralContainers
)
// DEPRECATED constants. Use the timeouts in framework.Framework instead.
@ -543,199 +534,6 @@ func RandomSuffix() string {
return strconv.Itoa(rand.Intn(10000))
}
// LookForStringInPodExec looks for the given string in the output of a command
// executed in the first container of specified pod.
// TODO(alejandrox1): move to pod/ subpkg once kubectl methods are refactored.
func LookForStringInPodExec(ns, podName string, command []string, expectedString string, timeout time.Duration) (result string, err error) {
return LookForStringInPodExecToContainer(ns, podName, "", command, expectedString, timeout)
}
// LookForStringInPodExecToContainer looks for the given string in the output of a
// command executed in specified pod container, or first container if not specified.
func LookForStringInPodExecToContainer(ns, podName, containerName string, command []string, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns)}
if len(containerName) > 0 {
args = append(args, fmt.Sprintf("--container=%s", containerName))
}
args = append(args, "--")
args = append(args, command...)
return RunKubectlOrDie(ns, args...)
})
}
// lookForString looks for the given string in the output of fn, repeatedly calling fn until
// the timeout is reached or the string is found. Returns last log and possibly
// error if the string was not found.
// TODO(alejandrox1): move to pod/ subpkg once kubectl methods are refactored.
func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) {
for t := time.Now(); time.Since(t) < timeout; time.Sleep(Poll) {
result = fn()
if strings.Contains(result, expectedString) {
return
}
}
err = fmt.Errorf("Failed to find \"%s\", last result: \"%s\"", expectedString, result)
return
}
// KubectlBuilder is used to build, customize and execute a kubectl Command.
// Add more functions to customize the builder as needed.
type KubectlBuilder struct {
cmd *exec.Cmd
timeout <-chan time.Time
}
// NewKubectlCommand returns a KubectlBuilder for running kubectl.
func NewKubectlCommand(namespace string, args ...string) *KubectlBuilder {
b := new(KubectlBuilder)
tk := e2ekubectl.NewTestKubeconfig(TestContext.CertDir, TestContext.Host, TestContext.KubeConfig, TestContext.KubeContext, TestContext.KubectlPath, namespace)
b.cmd = tk.KubectlCmd(args...)
return b
}
// WithEnv sets the given environment and returns itself.
func (b *KubectlBuilder) WithEnv(env []string) *KubectlBuilder {
b.cmd.Env = env
return b
}
// WithTimeout sets the given timeout and returns itself.
func (b *KubectlBuilder) WithTimeout(t <-chan time.Time) *KubectlBuilder {
b.timeout = t
return b
}
// WithStdinData sets the given data to stdin and returns itself.
func (b KubectlBuilder) WithStdinData(data string) *KubectlBuilder {
b.cmd.Stdin = strings.NewReader(data)
return &b
}
// WithStdinReader sets the given reader and returns itself.
func (b KubectlBuilder) WithStdinReader(reader io.Reader) *KubectlBuilder {
b.cmd.Stdin = reader
return &b
}
// ExecOrDie runs the kubectl executable or dies if error occurs.
func (b KubectlBuilder) ExecOrDie(namespace string) string {
str, err := b.Exec()
// In case of i/o timeout error, try talking to the apiserver again after 2s before dying.
// Note that we're still dying after retrying so that we can get visibility to triage it further.
if isTimeout(err) {
Logf("Hit i/o timeout error, talking to the server 2s later to see if it's temporary.")
time.Sleep(2 * time.Second)
retryStr, retryErr := RunKubectl(namespace, "version")
Logf("stdout: %q", retryStr)
Logf("err: %v", retryErr)
}
ExpectNoError(err)
return str
}
func isTimeout(err error) bool {
switch err := err.(type) {
case *url.Error:
if err, ok := err.Err.(net.Error); ok && err.Timeout() {
return true
}
case net.Error:
if err.Timeout() {
return true
}
}
return false
}
// Exec runs the kubectl executable.
func (b KubectlBuilder) Exec() (string, error) {
stdout, _, err := b.ExecWithFullOutput()
return stdout, err
}
// ExecWithFullOutput runs the kubectl executable, and returns the stdout and stderr.
func (b KubectlBuilder) ExecWithFullOutput() (string, string, error) {
var stdout, stderr bytes.Buffer
cmd := b.cmd
cmd.Stdout, cmd.Stderr = &stdout, &stderr
Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
if err := cmd.Start(); err != nil {
return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err)
}
errCh := make(chan error, 1)
go func() {
errCh <- cmd.Wait()
}()
select {
case err := <-errCh:
if err != nil {
var rc = 127
if ee, ok := err.(*exec.ExitError); ok {
rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus())
Logf("rc: %d", rc)
}
return stdout.String(), stderr.String(), uexec.CodeExitError{
Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err),
Code: rc,
}
}
case <-b.timeout:
b.cmd.Process.Kill()
return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr)
}
Logf("stderr: %q", stderr.String())
Logf("stdout: %q", stdout.String())
return stdout.String(), stderr.String(), nil
}
// RunKubectlOrDie is a convenience wrapper over kubectlBuilder
func RunKubectlOrDie(namespace string, args ...string) string {
return NewKubectlCommand(namespace, args...).ExecOrDie(namespace)
}
// RunKubectl is a convenience wrapper over kubectlBuilder
func RunKubectl(namespace string, args ...string) (string, error) {
return NewKubectlCommand(namespace, args...).Exec()
}
// RunKubectlWithFullOutput is a convenience wrapper over kubectlBuilder
// It will also return the command's stderr.
func RunKubectlWithFullOutput(namespace string, args ...string) (string, string, error) {
return NewKubectlCommand(namespace, args...).ExecWithFullOutput()
}
// RunKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func RunKubectlOrDieInput(namespace string, data string, args ...string) string {
return NewKubectlCommand(namespace, args...).WithStdinData(data).ExecOrDie(namespace)
}
// RunKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func RunKubectlInput(namespace string, data string, args ...string) (string, error) {
return NewKubectlCommand(namespace, args...).WithStdinData(data).Exec()
}
// RunKubemciWithKubeconfig is a convenience wrapper over RunKubemciCmd
func RunKubemciWithKubeconfig(args ...string) (string, error) {
if TestContext.KubeConfig != "" {
args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig)
}
return RunKubemciCmd(args...)
}
// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
// It assumes that kubemci exists in PATH.
func RunKubemciCmd(args ...string) (string, error) {
// kubemci is assumed to be in PATH.
kubemci := "kubemci"
b := new(KubectlBuilder)
args = append(args, "--gcp-project="+TestContext.CloudConfig.ProjectID)
b.cmd = exec.Command(kubemci, args...)
return b.Exec()
}
// StartCmdAndStreamOutput returns stdout and stderr after starting the given cmd.
func StartCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {
stdout, err = cmd.StdoutPipe()
@ -758,142 +556,6 @@ func TryKill(cmd *exec.Cmd) {
}
}
// testContainerOutputMatcher runs the given pod in the given namespace and waits
// for all of the containers in the podSpec to move into the 'Success' status, and tests
// the specified container log against the given expected output using the given matcher.
func (f *Framework) testContainerOutputMatcher(scenarioName string,
pod *v1.Pod,
containerIndex int,
expectedOutput []string,
matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) {
ginkgo.By(fmt.Sprintf("Creating a pod to test %v", scenarioName))
if containerIndex < 0 || containerIndex >= len(pod.Spec.Containers) {
Failf("Invalid container index: %d", containerIndex)
}
ExpectNoError(f.MatchContainerOutput(pod, pod.Spec.Containers[containerIndex].Name, expectedOutput, matcher))
}
// ContainerType signifies container type
type ContainerType int
const (
// Containers is for normal containers
Containers ContainerType = 1 << iota
// InitContainers is for init containers
InitContainers
// EphemeralContainers is for ephemeral containers
EphemeralContainers
)
// allFeatureEnabledContainers returns a ContainerType mask which includes all container
// types except for the ones guarded by feature gate.
// Copied from pkg/api/v1/pod to avoid pulling extra dependencies
func allFeatureEnabledContainers() ContainerType {
return AllContainers
}
// ContainerVisitor is called with each container spec, and returns true
// if visiting should continue.
// Copied from pkg/api/v1/pod to avoid pulling extra dependencies
type ContainerVisitor func(container *v1.Container, containerType ContainerType) (shouldContinue bool)
// visitContainers invokes the visitor function with a pointer to every container
// spec in the given pod spec with type set in mask. If visitor returns false,
// visiting is short-circuited. visitContainers returns true if visiting completes,
// false if visiting was short-circuited.
// Copied from pkg/api/v1/pod to avoid pulling extra dependencies
func visitContainers(podSpec *v1.PodSpec, mask ContainerType, visitor ContainerVisitor) bool {
if mask&InitContainers != 0 {
for i := range podSpec.InitContainers {
if !visitor(&podSpec.InitContainers[i], InitContainers) {
return false
}
}
}
if mask&Containers != 0 {
for i := range podSpec.Containers {
if !visitor(&podSpec.Containers[i], Containers) {
return false
}
}
}
if mask&EphemeralContainers != 0 {
for i := range podSpec.EphemeralContainers {
if !visitor((*v1.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon), EphemeralContainers) {
return false
}
}
}
return true
}
// MatchContainerOutput creates a pod and waits for all it's containers to exit with success.
// It then tests that the matcher with each expectedOutput matches the output of the specified container.
func (f *Framework) MatchContainerOutput(
pod *v1.Pod,
containerName string,
expectedOutput []string,
matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error {
ns := pod.ObjectMeta.Namespace
if ns == "" {
ns = f.Namespace.Name
}
podClient := f.PodClientNS(ns)
createdPod := podClient.Create(pod)
defer func() {
ginkgo.By("delete the pod")
podClient.DeleteSync(createdPod.Name, metav1.DeleteOptions{}, DefaultPodDeletionTimeout)
}()
// Wait for client pod to complete.
podErr := e2epod.WaitForPodSuccessInNamespaceTimeout(f.ClientSet, createdPod.Name, ns, f.Timeouts.PodStart)
// Grab its logs. Get host first.
podStatus, err := podClient.Get(context.TODO(), createdPod.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get pod status: %v", err)
}
if podErr != nil {
// Pod failed. Dump all logs from all containers to see what's wrong
_ = visitContainers(&podStatus.Spec, allFeatureEnabledContainers(), func(c *v1.Container, containerType ContainerType) bool {
logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, c.Name)
if err != nil {
Logf("Failed to get logs from node %q pod %q container %q: %v",
podStatus.Spec.NodeName, podStatus.Name, c.Name, err)
} else {
Logf("Output of node %q pod %q container %q: %s", podStatus.Spec.NodeName, podStatus.Name, c.Name, logs)
}
return true
})
return fmt.Errorf("expected pod %q success: %v", createdPod.Name, podErr)
}
Logf("Trying to get logs from node %s pod %s container %s: %v",
podStatus.Spec.NodeName, podStatus.Name, containerName, err)
// Sometimes the actual containers take a second to get started, try to get logs for 60s
logs, err := e2epod.GetPodLogs(f.ClientSet, ns, podStatus.Name, containerName)
if err != nil {
Logf("Failed to get logs from node %q pod %q container %q. %v",
podStatus.Spec.NodeName, podStatus.Name, containerName, err)
return fmt.Errorf("failed to get logs from %s for %s: %v", podStatus.Name, containerName, err)
}
for _, expected := range expectedOutput {
m := matcher(expected)
matches, err := m.Match(logs)
if err != nil {
return fmt.Errorf("expected %q in container output: %v", expected, err)
} else if !matches {
return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs))
}
}
return nil
}
// EventsLister is a func that lists events.
type EventsLister func(opts metav1.ListOptions, ns string) (*v1.EventList, error)
@ -1108,44 +770,6 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool
return true, nil
}
// RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec`
// inside of a shell.
func RunHostCmd(ns, name, cmd string) (string, error) {
return RunKubectl(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd)
}
// RunHostCmdWithFullOutput runs the given cmd in the context of the given pod using `kubectl exec`
// inside of a shell. It will also return the command's stderr.
func RunHostCmdWithFullOutput(ns, name, cmd string) (string, string, error) {
return RunKubectlWithFullOutput(ns, "exec", name, "--", "/bin/sh", "-x", "-c", cmd)
}
// RunHostCmdOrDie calls RunHostCmd and dies on error.
func RunHostCmdOrDie(ns, name, cmd string) string {
stdout, err := RunHostCmd(ns, name, cmd)
Logf("stdout: %v", stdout)
ExpectNoError(err)
return stdout
}
// RunHostCmdWithRetries calls RunHostCmd and retries all errors
// until it succeeds or the specified timeout expires.
// This can be used with idempotent commands to deflake transient Node issues.
func RunHostCmdWithRetries(ns, name, cmd string, interval, timeout time.Duration) (string, error) {
start := time.Now()
for {
out, err := RunHostCmd(ns, name, cmd)
if err == nil {
return out, nil
}
if elapsed := time.Since(start); elapsed > timeout {
return out, fmt.Errorf("RunHostCmd still failed after %v: %v", elapsed, err)
}
Logf("Waiting %v to retry failed RunHostCmd: %v", interval, err)
time.Sleep(interval)
}
}
// AllNodesReady checks whether all registered nodes are ready. Setting -1 on
// TestContext.AllowedNotReadyNodes will bypass the post test node readiness check.
// TODO: we should change the AllNodesReady call in AfterEach to WaitForAllNodesHealthy,
@ -1194,13 +818,6 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error {
return nil
}
// LookForStringInLog looks for the given string in the log of a specific pod container
func LookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return RunKubectlOrDie(ns, "logs", podName, container)
})
}
// EnsureLoadBalancerResourcesDeleted ensures that cloud load balancer resources that were created
// are actually cleaned up. Currently only implemented for GCE/GKE.
func EnsureLoadBalancerResourcesDeleted(ip, portRange string) error {
@ -1323,25 +940,6 @@ func GetControlPlaneAddresses(c clientset.Interface) []string {
return ips.List()
}
// CreateEmptyFileOnPod creates empty file at given path on the pod.
// TODO(alejandrox1): move to subpkg pod once kubectl methods have been refactored.
func CreateEmptyFileOnPod(namespace string, podName string, filePath string) error {
_, err := RunKubectl(namespace, "exec", podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath))
return err
}
// DumpDebugInfo dumps debug info of tests.
func DumpDebugInfo(c clientset.Interface, ns string) {
sl, _ := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: labels.Everything().String()})
for _, s := range sl.Items {
desc, _ := RunKubectl(ns, "describe", "po", s.Name)
Logf("\nOutput of kubectl describe %v:\n%v", s.Name, desc)
l, _ := RunKubectl(ns, "logs", s.Name, "--tail=100")
Logf("\nLast 100 log lines of %v:\n%v", s.Name, l)
}
}
// PrettyPrintJSON converts metrics to JSON format.
func PrettyPrintJSON(metrics interface{}) string {
output := &bytes.Buffer{}

View File

@ -57,6 +57,8 @@ import (
clientexec "k8s.io/client-go/util/exec"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2etodokubectl "k8s.io/kubernetes/test/e2e/framework/todo/kubectl"
e2etodopod "k8s.io/kubernetes/test/e2e/framework/todo/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
uexec "k8s.io/utils/exec"
@ -354,7 +356,7 @@ func startVolumeServer(client clientset.Interface, config TestConfig) *v1.Pod {
}
}
if config.ServerReadyMessage != "" {
_, err := framework.LookForStringInLog(pod.Namespace, pod.Name, serverPodName, config.ServerReadyMessage, VolumeServerPodStartupTimeout)
_, err := e2etodopod.LookForStringInLog(pod.Namespace, pod.Name, serverPodName, config.ServerReadyMessage, VolumeServerPodStartupTimeout)
framework.ExpectNoError(err, "Failed to find %q in pod logs: %s", config.ServerReadyMessage, err)
}
return pod
@ -475,7 +477,7 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string
// Block: check content
deviceName := fmt.Sprintf("/opt/%d", i)
commands := GenerateReadBlockCmd(deviceName, len(test.ExpectedContent))
_, err := framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute)
_, err := e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute)
framework.ExpectNoError(err, "failed: finding the contents of the block device %s.", deviceName)
// Check that it's a real block device
@ -484,7 +486,7 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string
// Filesystem: check content
fileName := fmt.Sprintf("/opt/%d/%s", i, test.File)
commands := GenerateReadFileCmd(fileName)
_, err := framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute)
_, err := e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, commands, test.ExpectedContent, time.Minute)
framework.ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName)
// Check that a directory has been mounted
@ -495,14 +497,14 @@ func testVolumeContent(f *framework.Framework, pod *v1.Pod, containerName string
// Filesystem: check fsgroup
if fsGroup != nil {
ginkgo.By("Checking fsGroup is correct.")
_, err = framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"ls", "-ld", dirName}, strconv.Itoa(int(*fsGroup)), time.Minute)
_, err = e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"ls", "-ld", dirName}, strconv.Itoa(int(*fsGroup)), time.Minute)
framework.ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
}
// Filesystem: check fsType
if fsType != "" {
ginkgo.By("Checking fsType is correct.")
_, err = framework.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"grep", " " + dirName + " ", "/proc/mounts"}, fsType, time.Minute)
_, err = e2etodopod.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, containerName, []string{"grep", " " + dirName + " ", "/proc/mounts"}, fsType, time.Minute)
framework.ExpectNoError(err, "failed: getting the right fsType %s", fsType)
}
}
@ -587,7 +589,7 @@ func InjectContent(f *framework.Framework, config TestConfig, fsGroup *int64, fs
fileName := fmt.Sprintf("/opt/%d/%s", i, test.File)
commands = append(commands, generateWriteFileCmd(test.ExpectedContent, fileName)...)
}
out, err := framework.RunKubectl(injectorPod.Namespace, commands...)
out, err := e2etodokubectl.RunKubectl(injectorPod.Namespace, commands...)
framework.ExpectNoError(err, "failed: writing the contents: %s", out)
}