diff --git a/test/e2e/auth/projected_clustertrustbundle.go b/test/e2e/auth/projected_clustertrustbundle.go index 4213da9a4ed..657c5614edd 100644 --- a/test/e2e/auth/projected_clustertrustbundle.go +++ b/test/e2e/auth/projected_clustertrustbundle.go @@ -23,15 +23,21 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "math/big" + mathrand "math/rand/v2" "os" "regexp" + "time" certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" @@ -42,88 +48,494 @@ import ( "github.com/onsi/ginkgo/v2" ) +const ( + testSignerOneName = "test.test/signer-one" + testSignerTwoName = "test.test/signer-two" + aliveSignersKey = "signer.alive=true" + deadSignersKey = "signer.alive=false" + noSignerKey = "no-signer" +) + var _ = SIGDescribe(feature.ClusterTrustBundle, feature.ClusterTrustBundleProjection, func() { f := framework.NewDefaultFramework("projected-clustertrustbundle") f.NamespacePodSecurityLevel = admissionapi.LevelBaseline - goodCert1 := mustMakeCertificate(&x509.Certificate{ + initCTBs, pemMapping := initCTBData() + + ginkgo.JustBeforeEach(func(ctx context.Context) { + cleanup := mustInitCTBs(ctx, f, initCTBs) + ginkgo.DeferCleanup(cleanup) + }) + + ginkgo.It("should be able to mount a single ClusterTrustBundle by name", func(ctx context.Context) { + + for _, tt := range []struct { + name string + ctbName string + optional *bool + expectedOutput []string + }{ + { + name: "name of an existing CTB", + ctbName: "test.test.signer-one.4", + expectedOutput: expectedRegexFromPEMs(initCTBs[4].Spec.TrustBundle), + }, + { + name: "name of a CTB that does not exist + optional=true", + ctbName: "does-not-exist.at.all", + optional: ptr.To(true), + expectedOutput: []string{"content of file \"/var/run/ctbtest/trust-anchors.pem\": \n$"}, + }, + } { + pod := podForCTBProjection(v1.VolumeProjection{ + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Name: &tt.ctbName, + Path: "trust-anchors.pem", + Optional: tt.optional, + }, + }) + + fileModeRegexp := getFileModeRegex("/var/run/ctbtest/trust-anchors.pem", nil) + expectedOutput := append(tt.expectedOutput, fileModeRegexp) + + e2epodoutput.TestContainerOutputRegexp(ctx, f, "project cluster trust bundle", pod, 0, expectedOutput) + } + }) + + ginkgo.Describe("should be capable to mount multiple trust bundles by signer+labels", func() { + fileModeRegexp := getFileModeRegex("/var/run/ctbtest/trust-bundle.crt", nil) + + for _, tt := range []struct { + name string + signerName string + selector *metav1.LabelSelector + optionalVolume *bool + expectedOutputRegex []string + }{ + { + name: "can combine multiple CTBs with signer name and label selector", + signerName: testSignerOneName, + selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "signer.alive": "true", + }, + }, + expectedOutputRegex: expectedRegexFromPEMs(pemMapping[testSignerOneName].Intersection(pemMapping[aliveSignersKey]).UnsortedList()...), + }, + { + name: "should start if only signer name and nil label selector + optional=true", + signerName: testSignerOneName, + selector: nil, // == match nothing + optionalVolume: ptr.To(true), + expectedOutputRegex: []string{"content of file \"/var/run/ctbtest/trust-bundle.crt\": \n$"}, + }, + { + name: "should start if only signer name and explicit label selector matches nothing + optional=true", + signerName: testSignerOneName, + selector: &metav1.LabelSelector{MatchLabels: map[string]string{"thismatches": "nothing"}}, + optionalVolume: ptr.To(true), + expectedOutputRegex: []string{"content of file \"/var/run/ctbtest/trust-bundle.crt\": \n$"}, + }, + { + name: "can combine all signer CTBs with an empty label selector", + signerName: testSignerOneName, + selector: &metav1.LabelSelector{}, + expectedOutputRegex: expectedRegexFromPEMs(pemMapping[testSignerOneName].UnsortedList()...), + }, + } { + ginkgo.It(tt.name, func(ctx context.Context) { + pod := podForCTBProjection(v1.VolumeProjection{ + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Path: "trust-bundle.crt", + SignerName: &tt.signerName, + LabelSelector: tt.selector, + Optional: tt.optionalVolume, + }, + }) + + expectedOutput := append(tt.expectedOutputRegex, fileModeRegexp) + + e2epodoutput.TestContainerOutputRegexp(ctx, f, "project cluster trust bundle", pod, 0, expectedOutput) + }) + } + }) + + ginkgo.Describe("should prevent a pod from starting if: ", func() { + + for _, tt := range []struct { + name string + ctb *v1.ClusterTrustBundleProjection + }{ + { + name: "sets optional=false and no trust bundle matches query", + ctb: &v1.ClusterTrustBundleProjection{ + Optional: ptr.To(false), + Path: "trust-bundle.crt", + SignerName: ptr.To(testSignerOneName), + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "signer.alive": "unknown", + }, + }, + }, + }, + { + name: "sets optional=false and the configured CTB does not exist", + ctb: &v1.ClusterTrustBundleProjection{ + Optional: ptr.To(false), + Path: "trust-bundle.crt", + Name: ptr.To("does-not-exist"), + }, + }, + } { + ginkgo.It(tt.name, func(ctx context.Context) { + pod := podForCTBProjection(v1.VolumeProjection{ClusterTrustBundle: tt.ctb}) + + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + framework.Failf("failed to create a testing container: %v", err) + } + + volumeNotReady := false + var latestReadyStatus *v1.PodCondition + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, true, func(waitCtx context.Context) (done bool, err error) { + waitPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(waitCtx, pod.Name, metav1.GetOptions{}) + if err != nil { + framework.Logf("failed to get pod: %v", err) + return false, nil + } + + if waitPod.Status.Phase == v1.PodRunning { + return true, nil + } + + if latestReadyStatus = podutil.GetPodReadyCondition(waitPod.Status); latestReadyStatus != nil && + latestReadyStatus.Status == v1.ConditionFalse && + latestReadyStatus.Reason == "ContainersNotReady" && + latestReadyStatus.Message == "containers with unready status: [projected-ctb-volume-test-0]" { + volumeNotReady = true + return false, nil + } + volumeNotReady = false + + return false, nil + }) + + if err == nil { + framework.Fail("expected the pod not to start running, but it did") + } else if !errors.Is(err, context.DeadlineExceeded) { + framework.Failf("expected deadline exceeded, but got: %v", err) + } + + if !volumeNotReady { + framework.Failf("expected the pod to not be ready because of a missing volume, but its status is different: %v", latestReadyStatus) + } + }) + } + }) + ginkgo.It("should be able to specify multiple CTB volumes", func(ctx context.Context) { + pod := podForCTBProjection( + v1.VolumeProjection{ + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Name: ptr.To("test.test.signer-one.4"), + Path: "trust-anchors.pem", + }, + }, + v1.VolumeProjection{ + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Path: "trust-bundle.crt", + SignerName: ptr.To(testSignerOneName), + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "signer.alive": "false", + }, + }, + }, + }, + ) + expectedOutputs := map[int][]string{ + 0: append(expectedRegexFromPEMs(pemMapping[noSignerKey].UnsortedList()...), getFileModeRegex("/var/run/ctbtest/trust-anchors.pem", nil)), + 1: append(expectedRegexFromPEMs(pemMapping[testSignerOneName].Intersection(pemMapping[deadSignersKey]).UnsortedList()...), getFileModeRegex("/var/run/ctbtest/trust-bundle.crt", nil)), + } + + e2epodoutput.TestContainerOutputsRegexp(ctx, f, "multiple CTB volumes", pod, expectedOutputs) + }) + + ginkgo.It("should be able to mount a big number (>100) of CTBs", func(ctx context.Context) { + const numCTBs = 150 + + var initCTBs []*certificatesv1alpha1.ClusterTrustBundle + var cleanups []func(ctx context.Context) + var projections []v1.VolumeProjection + + defer func() { + for _, c := range cleanups { + c(ctx) + } + }() + for i := range numCTBs { + ctb := ctbForCA(fmt.Sprintf("test.test:signer-hundreds:%d", i), "test.test/signer-hundreds", mustMakeCAPEM(fmt.Sprintf("root%d", i)), nil) + initCTBs = append(initCTBs, ctb) + cleanups = append(cleanups, mustCreateCTB(ctx, f, ctb)) + projections = append(projections, v1.VolumeProjection{ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ // TODO: maybe mount them all to a single pod? + Name: ptr.To(fmt.Sprintf("test.test:signer-hundreds:%d", i)), + Path: fmt.Sprintf("trust-anchors-%d.pem", i), + }, + }) + } + + ginkgo.By("as a single projection with many sources", func() { + randomIndexToTest := mathrand.Int32N(numCTBs) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pod-projected-ctb-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "projected-ctb-volume-test", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{ + "mounttest", + fmt.Sprintf("--file_content=/var/run/ctbtest/trust-anchors-%d.pem", randomIndexToTest), + fmt.Sprintf("--file_mode=/var/run/ctbtest/trust-anchors-%d.pem", randomIndexToTest), + }, + VolumeMounts: []v1.VolumeMount{{ + Name: "ctb-volume", + MountPath: "/var/run/ctbtest", + }}, + }, + }, + Volumes: []v1.Volume{{ + Name: "ctb-volume", + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: projections, + }, + }, + }}, + }, + } + + expectedOutputs := append(expectedRegexFromPEMs(initCTBs[randomIndexToTest].Spec.TrustBundle), getFileModeRegex(fmt.Sprintf("/var/run/ctbtest/trust-anchors-%d.pem", randomIndexToTest), nil)) + e2epodoutput.TestContainerOutputRegexp(ctx, f, "single CTB volume with many files", pod, 0, expectedOutputs) + }) + + ginkgo.By("as separate projections", func() { + randomIndexToTest := mathrand.Int32N(numCTBs) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pod-projected-ctb-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "projected-ctb-volume-test", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{ + "mounttest", + fmt.Sprintf("--file_content=/var/run/ctbtest-%d/%s", randomIndexToTest, projections[randomIndexToTest].ClusterTrustBundle.Path), + fmt.Sprintf("--file_mode=/var/run/ctbtest-%d/%s", randomIndexToTest, projections[randomIndexToTest].ClusterTrustBundle.Path), + }, + }, + }, + }, + } + for i := range projections { + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: fmt.Sprintf("ctb-volume-%d", i), + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{projections[i]}, + }, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ + Name: fmt.Sprintf("ctb-volume-%d", i), + MountPath: fmt.Sprintf("/var/run/ctbtest-%d", i), + }) + } + + expectedOutputs := append(expectedRegexFromPEMs(initCTBs[randomIndexToTest].Spec.TrustBundle), getFileModeRegex(fmt.Sprintf("/var/run/ctbtest-%d/trust-anchors-%d.pem", randomIndexToTest, randomIndexToTest), nil)) + e2epodoutput.TestContainerOutputRegexp(ctx, f, "many CTB volumes", pod, 0, expectedOutputs) + }) + + ginkgo.By("as a single projection joined in a single file by signer name", func() { + pod := podForCTBProjection(v1.VolumeProjection{ + ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ + Path: "trust-anchors.pem", + SignerName: ptr.To("test.test/signer-hundreds"), + LabelSelector: &metav1.LabelSelector{}, // == match everything + }, + }) + + expectedOutputs := append(expectedRegexFromPEMs(ctbsToPEMs(initCTBs)...), getFileModeRegex("/var/run/ctbtest/trust-anchors.pem", nil)) + e2epodoutput.TestContainerOutputRegexp(ctx, f, "single CTB volume with a single file", pod, 0, expectedOutputs) + + }) + }) +}) + +func expectedRegexFromPEMs(certPEMs ...string) []string { + var ret []string + for _, pem := range certPEMs { + ret = append(ret, regexp.QuoteMeta(pem)) + } + return ret +} + +func podForCTBProjection(projectionSources ...v1.VolumeProjection) *v1.Pod { + const volumeNameFmt = "ctb-volume-%d" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-projected-ctb-" + string(uuid.NewUUID()), + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + }, + } + + for i := range projectionSources { + pod.Spec.Containers = append(pod.Spec.Containers, + v1.Container{ + Name: fmt.Sprintf("projected-ctb-volume-test-%d", i), + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{ + "mounttest", + fmt.Sprintf("--file_content=/var/run/ctbtest/%s", projectionSources[i].ClusterTrustBundle.Path), + fmt.Sprintf("--file_mode=/var/run/ctbtest/%s", projectionSources[i].ClusterTrustBundle.Path), + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: fmt.Sprintf(volumeNameFmt, i), + MountPath: "/var/run/ctbtest", + }, + }, + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, + v1.Volume{ + Name: fmt.Sprintf(volumeNameFmt, i), + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{projectionSources[i]}, + }, + }, + }) + } + + return pod +} + +// mustInitCTBs creates a testSet of ClusterTrustBundles and spreads them into several +// categories based on their signer name and labels. +// It returns a cleanup function for all the ClusterTrustBundle objects it created. +// It also returns a map of sets of PEMs like so: +// +// { +// "test.test/signer-one": , +// "test.test/signer-two": , +// "signer.alive=true": , +// "signer.alive=false": , +// "no-signer": , +// } +func initCTBData() ([]*certificatesv1alpha1.ClusterTrustBundle, map[string]sets.Set[string]) { + var pemSets = map[string]sets.Set[string]{ + testSignerOneName: sets.New[string](), + testSignerTwoName: sets.New[string](), + aliveSignersKey: sets.New[string](), + deadSignersKey: sets.New[string](), + noSignerKey: sets.New[string](), + } + + var ctbs []*certificatesv1alpha1.ClusterTrustBundle + + for i := range 10 { + caPEM := mustMakeCAPEM(fmt.Sprintf("root%d", i)) + + switch i { + case 1, 2, 3: + ctbs = append(ctbs, ctbForCA(fmt.Sprintf("test.test:signer-one:%d", i), testSignerOneName, caPEM, map[string]string{"signer.alive": "true"})) + + pemSets[testSignerOneName].Insert(caPEM) + pemSets[aliveSignersKey].Insert(caPEM) + case 4: + ctbs = append(ctbs, ctbForCA(fmt.Sprintf("test.test.signer-one.%d", i), "", caPEM, map[string]string{"signer.alive": "true"})) + + pemSets[noSignerKey].Insert(caPEM) + case 5: + ctbs = append(ctbs, ctbForCA(fmt.Sprintf("test.test:signer-two:%d", i), testSignerTwoName, caPEM, map[string]string{"signer.alive": "true"})) + + pemSets[testSignerTwoName].Insert(caPEM) + pemSets[aliveSignersKey].Insert(caPEM) + case 6, 7: + ctbs = append(ctbs, ctbForCA(fmt.Sprintf("test.test:signer-one:%d", i), testSignerOneName, caPEM, map[string]string{"signer.alive": "false"})) + + pemSets[testSignerOneName].Insert(caPEM) + pemSets[deadSignersKey].Insert(caPEM) + default: // 0, 8 ,9 + ctbs = append(ctbs, ctbForCA(fmt.Sprintf("test.test:signer-one:%d", i), testSignerOneName, caPEM, nil)) + + pemSets[testSignerOneName].Insert(caPEM) + } + } + + return ctbs, pemSets +} + +func ctbForCA(ctbName, signerName, caPEM string, labels map[string]string) *certificatesv1alpha1.ClusterTrustBundle { + return &certificatesv1alpha1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctbName, + Labels: labels, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: signerName, + TrustBundle: caPEM, + }, + } +} + +func mustInitCTBs(ctx context.Context, f *framework.Framework, ctbs []*certificatesv1alpha1.ClusterTrustBundle) func(context.Context) { + cleanups := []func(context.Context){} + for _, ctb := range ctbs { + ctb := ctb + cleanups = append(cleanups, mustCreateCTB(ctx, f, ctb)) + } + + return func(ctx context.Context) { + for _, c := range cleanups { + c(ctx) + } + } +} + +func mustCreateCTB(ctx context.Context, f *framework.Framework, ctb *certificatesv1alpha1.ClusterTrustBundle) func(context.Context) { + if _, err := f.ClientSet.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, ctb, metav1.CreateOptions{}); err != nil { + framework.Failf("Error while creating ClusterTrustBundle: %v", err) + } + + return func(ctx context.Context) { + if err := f.ClientSet.CertificatesV1alpha1().ClusterTrustBundles().Delete(ctx, ctb.Name, metav1.DeleteOptions{}); err != nil { + framework.Logf("failed to remove a cluster trust bundle: %v", err) + } + } +} + +func mustMakeCAPEM(cn string) string { + asnCert := mustMakeCertificate(&x509.Certificate{ SerialNumber: big.NewInt(0), Subject: pkix.Name{ - CommonName: "root1", + CommonName: cn, }, IsCA: true, BasicConstraintsValid: true, }) - goodCert1Block := string(mustMakePEMBlock("CERTIFICATE", nil, goodCert1)) - - ginkgo.It("should be able to mount a single ClusterTrustBundle by name", func(ctx context.Context) { - - ctb1 := &certificatesv1alpha1.ClusterTrustBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ctb1", - }, - Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ - TrustBundle: goodCert1Block, - }, - } - - if _, err := f.ClientSet.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, ctb1, metav1.CreateOptions{}); err != nil { - framework.Failf("Error while creating ClusterTrustBundle: %v", err) - } - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-projected-ctb-" + string(uuid.NewUUID()), - }, - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - Containers: []v1.Container{ - { - Name: "projected-ctb-volume-test", - Image: imageutils.GetE2EImage(imageutils.Agnhost), - Args: []string{ - "mounttest", - "--file_content=/var/run/ctbtest/trust-anchors.pem", - "--file_mode=/var/run/ctbtest/trust-anchors.pem", - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "ctb-volume", - MountPath: "/var/run/ctbtest", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ctb-volume", - VolumeSource: v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - ClusterTrustBundle: &v1.ClusterTrustBundleProjection{ - Name: ptr.To("ctb1"), - Path: "trust-anchors.pem", - }, - }, - }, - }, - }, - }, - }, - }, - } - - fileModeRegexp := getFileModeRegex("/var/run/ctbtest/trust-anchors.pem", nil) - expectedOutput := []string{ - regexp.QuoteMeta(goodCert1Block), - fileModeRegexp, - } - - e2epodoutput.TestContainerOutputRegexp(ctx, f, "project cluster trust bundle", pod, 0, expectedOutput) - }) -}) + return mustMakePEMBlock("CERTIFICATE", nil, asnCert) +} func mustMakeCertificate(template *x509.Certificate) []byte { pub, priv, err := ed25519.GenerateKey(rand.Reader) @@ -167,3 +579,11 @@ func getFileModeRegex(filePath string, mask *int32) string { return fmt.Sprintf("(%s|%s)", linuxOutput, windowsOutput) } + +func ctbsToPEMs(ctbs []*certificatesv1alpha1.ClusterTrustBundle) []string { + var certPEMs []string + for _, ctb := range ctbs { + certPEMs = append(certPEMs, ctb.Spec.TrustBundle) + } + return certPEMs +} diff --git a/test/e2e/framework/pod/output/output.go b/test/e2e/framework/pod/output/output.go index 8354634329d..b649f99c1fc 100644 --- a/test/e2e/framework/pod/output/output.go +++ b/test/e2e/framework/pod/output/output.go @@ -157,6 +157,16 @@ func MatchContainerOutput( containerName string, expectedOutput []string, matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error { + + return MatchMultipleContainerOutputs(ctx, f, pod, map[string][]string{containerName: expectedOutput}, matcher) +} + +func MatchMultipleContainerOutputs( + ctx context.Context, + f *framework.Framework, + pod *v1.Pod, + expectedOutputs map[string][]string, // map of container name -> expected outputs + matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) error { ns := pod.ObjectMeta.Namespace if ns == "" { ns = f.Namespace.Name @@ -193,24 +203,26 @@ func MatchContainerOutput( return fmt.Errorf("expected pod %q success: %v", createdPod.Name, podErr) } - framework.Logf("Trying to get logs from node %s pod %s container %s: %v", - podStatus.Spec.NodeName, podStatus.Name, containerName, err) + for cName, expectedOutput := range expectedOutputs { + framework.Logf("Trying to get logs from node %s pod %s container %s: %v", + podStatus.Spec.NodeName, podStatus.Name, cName, err) - // Sometimes the actual containers take a second to get started, try to get logs for 60s - logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, ns, podStatus.Name, containerName) - if err != nil { - framework.Logf("Failed to get logs from node %q pod %q container %q. %v", - podStatus.Spec.NodeName, podStatus.Name, containerName, err) - return fmt.Errorf("failed to get logs from %s for %s: %w", podStatus.Name, containerName, err) - } - - for _, expected := range expectedOutput { - m := matcher(expected) - matches, err := m.Match(logs) + // Sometimes the actual containers take a second to get started, try to get logs for 60s + logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, ns, podStatus.Name, cName) if err != nil { - return fmt.Errorf("expected %q in container output: %w", expected, err) - } else if !matches { - return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs)) + framework.Logf("Failed to get logs from node %q pod %q container %q. %v", + podStatus.Spec.NodeName, podStatus.Name, cName, err) + return fmt.Errorf("failed to get logs from %s for %s: %w", podStatus.Name, cName, err) + } + + for _, expected := range expectedOutput { + m := matcher(expected) + matches, err := m.Match(logs) + if err != nil { + return fmt.Errorf("expected %q in container output: %w", expected, err) + } else if !matches { + return fmt.Errorf("expected %q in container output: %s", expected, m.FailureMessage(logs)) + } } } @@ -228,7 +240,11 @@ func TestContainerOutput(ctx context.Context, f *framework.Framework, scenarioNa // for all of the containers in the podSpec to move into the 'Success' status, and tests // the specified container log against the given expected output using a regexp matcher. func TestContainerOutputRegexp(ctx context.Context, f *framework.Framework, scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) { - TestContainerOutputMatcher(ctx, f, scenarioName, pod, containerIndex, expectedOutput, gomega.MatchRegexp) + TestContainerOutputsRegexp(ctx, f, scenarioName, pod, map[int][]string{containerIndex: expectedOutput}) +} + +func TestContainerOutputsRegexp(ctx context.Context, f *framework.Framework, scenarioName string, pod *v1.Pod, expectedOutputs map[int][]string) { + TestContainerOutputsMatcher(ctx, f, scenarioName, pod, expectedOutputs, gomega.MatchRegexp) } // TestContainerOutputMatcher runs the given pod in the given namespace and waits @@ -246,3 +262,23 @@ func TestContainerOutputMatcher(ctx context.Context, f *framework.Framework, } framework.ExpectNoError(MatchContainerOutput(ctx, f, pod, pod.Spec.Containers[containerIndex].Name, expectedOutput, matcher)) } + +func TestContainerOutputsMatcher(ctx context.Context, f *framework.Framework, + scenarioName string, + pod *v1.Pod, + expectedOutputs map[int][]string, + matcher func(string, ...interface{}) gomegatypes.GomegaMatcher) { + + ginkgo.By(fmt.Sprintf("Creating a pod to test %v", scenarioName)) + + expectedNameOutputs := make(map[string][]string, len(expectedOutputs)) + for containerIndex, expectedOutput := range expectedOutputs { + expectedOutput := expectedOutput + if containerIndex < 0 || containerIndex >= len(pod.Spec.Containers) { + framework.Failf("Invalid container index: %d", containerIndex) + } + expectedNameOutputs[pod.Spec.Containers[containerIndex].Name] = expectedOutput + } + framework.ExpectNoError(MatchMultipleContainerOutputs(ctx, f, pod, expectedNameOutputs, matcher)) + +}