From 123ab8033359f87ad89c87f9dca04a1f324d0ec0 Mon Sep 17 00:00:00 2001 From: Todd Neal Date: Wed, 22 Feb 2023 20:38:15 -0600 Subject: [PATCH] add more init container lifetime testing Add some additional init container tests that work via monitoring container lifetime based on logs written to a common file. This allows more easily writing assertions about the container lifetimes with respect to one another. --- test/e2e_node/init_containers_test.go | 345 ++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 test/e2e_node/init_containers_test.go diff --git a/test/e2e_node/init_containers_test.go b/test/e2e_node/init_containers_test.go new file mode 100644 index 00000000000..cbf522f2099 --- /dev/null +++ b/test/e2e_node/init_containers_test.go @@ -0,0 +1,345 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + admissionapi "k8s.io/pod-security-admission/api" + + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" +) + +type containerType byte + +const ( + containerTypeInvalid containerType = iota + containerTypeInit + containerTypeRegular +) + +type containerTestConfig struct { + Name string + Type containerType + ExitCode int + Delay int +} + +func (c containerTestConfig) Command() []string { + var cmd bytes.Buffer + // all outputs are in the format of: + // timestamp container-name message + fmt.Fprintf(&cmd, "echo `date +%%s` '%s Starting' >> /shared/output; ", c.Name) + fmt.Fprintf(&cmd, "echo `date +%%s` '%s Delaying %d' >> /shared/output; ", c.Name, c.Delay) + if c.Delay != 0 { + fmt.Fprintf(&cmd, "sleep %d; ", c.Delay) + } + fmt.Fprintf(&cmd, "echo `date +%%s` '%s Exiting' >> /shared/output; ", c.Name) + fmt.Fprintf(&cmd, "exit %d", c.ExitCode) + return []string{"sh", "-c", cmd.String()} +} + +var _ = SIGDescribe("InitContainers [NodeConformance]", func() { + f := framework.NewDefaultFramework("initcontainers-test") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + var tmpDir string + ginkgo.BeforeEach(func() { + var err error + tmpDir, err = os.MkdirTemp("", "init-container-*") + framework.ExpectNoError(err, "creating temp directory") + + }) + ginkgo.AfterEach(func() { + os.RemoveAll(tmpDir) + }) + + ginkgo.It("should launch init container serially before a regular container", func() { + init1 := containerTestConfig{ + Name: "init-1", + Type: containerTypeInit, + Delay: 1, + ExitCode: 0, + } + init2 := containerTestConfig{ + Name: "init-2", + Type: containerTypeInit, + Delay: 1, + ExitCode: 0, + } + init3 := containerTestConfig{ + Name: "init-3", + Type: containerTypeInit, + Delay: 1, + ExitCode: 0, + } + regular1 := containerTestConfig{ + Name: "regular-1", + Type: containerTypeRegular, + Delay: 1, + ExitCode: 0, + } + + /// generates an out file output like: + // + // 1677116487 init-1 Starting + // 1677116487 init-1 Delaying 1 + // 1677116488 init-1 Exiting + // 1677116489 init-2 Starting + // 1677116489 init-2 Delaying 1 + // 1677116490 init-2 Exiting + // 1677116491 init-3 Starting + // 1677116491 init-3 Delaying 1 + // 1677116492 init-3 Exiting + // 1677116493 regular-1 Starting + // 1677116493 regular-1 Delaying 1 + // 1677116494 regular-1 Exiting + + podSpec := getContainerOrderingPod("initcontainer-test-pod", + tmpDir, init1, init2, init3, regular1) + + podSpec = e2epod.NewPodClient(f).Create(context.TODO(), podSpec) + ginkgo.By("Waiting for the pod to finish") + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By("Parsing results") + results := parseOutput(tmpDir) + + // which we then use to make assertions regarding container ordering + ginkgo.By("Analyzing results") + framework.ExpectNoError(results.StartsBefore(init1, init2)) + framework.ExpectNoError(results.ExitsBefore(init1, init2)) + + framework.ExpectNoError(results.StartsBefore(init2, init3)) + framework.ExpectNoError(results.ExitsBefore(init2, init3)) + + framework.ExpectNoError(results.StartsBefore(init3, regular1)) + framework.ExpectNoError(results.ExitsBefore(init3, regular1)) + }) + + ginkgo.It("should not launch regular containers if an init container fails", func() { + init1 := containerTestConfig{ + Name: "init-1", + Type: containerTypeInit, + Delay: 1, + ExitCode: 1, + } + regular1 := containerTestConfig{ + Name: "regular-1", + Type: containerTypeRegular, + Delay: 1, + ExitCode: 0, + } + + podSpec := getContainerOrderingPod("initcontainer-test-pod-failure", + tmpDir, init1, regular1) + + podSpec = e2epod.NewPodClient(f).Create(context.TODO(), podSpec) + ginkgo.By("Waiting for the pod to fail") + err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By("Parsing results") + results := parseOutput(tmpDir) + + ginkgo.By("Analyzing results") + // init container should start and exit with an error, and the regular container should never start + framework.ExpectNoError(results.Starts(init1)) + framework.ExpectNoError(results.Exits(init1)) + framework.ExpectNoError(results.DoesntStart(regular1)) + }) + +}) + +type containerOutput struct { + line int + timestamp string + containerName string + command string +} +type containerOutputList []containerOutput + +func (o containerOutputList) String() string { + var b bytes.Buffer + for _, v := range o { + fmt.Fprintf(&b, "%d %s %s %s\n", v.line, v.timestamp, v.containerName, v.command) + } + return b.String() +} + +// StartsBefore returns an error if lhs did not start before rhs +func (o containerOutputList) StartsBefore(lhs, rhs containerTestConfig) error { + lhsStart := o.findIndex(lhs.Name, "Starting") + rhsStart := o.findIndex(rhs.Name, "Starting") + + if lhsStart == -1 { + return fmt.Errorf("couldn't find that container %s ever started, got %s", lhs.Name, o) + } + if rhsStart == -1 { + return fmt.Errorf("couldn't find that container %s ever started, got %s", rhs.Name, o) + } + if lhsStart >= rhsStart { + return fmt.Errorf("expected container %s to start before %s, got %s", lhs.Name, rhs.Name, o) + } + return nil +} + +// ExitsBefore returns an error if lhs did not end before rhs +func (o containerOutputList) ExitsBefore(lhs, rhs containerTestConfig) error { + lhsExit := o.findIndex(lhs.Name, "Exiting") + rhsExit := o.findIndex(rhs.Name, "Exiting") + + if lhsExit == -1 { + return fmt.Errorf("couldn't find that container %s ever exited, got %s", lhs.Name, o) + } + if rhsExit == -1 { + return fmt.Errorf("couldn't find that container %s ever exited, got %s", rhs.Name, o) + } + if lhsExit >= rhsExit { + return fmt.Errorf("expected container %s to exit before %s, got %s", lhs.Name, rhs.Name, o) + } + return nil +} + +// Starts returns an error if the container was not found to have started +func (o containerOutputList) Starts(c containerTestConfig) error { + if idx := o.findIndex(c.Name, "Starting"); idx == -1 { + return fmt.Errorf("couldn't find that container %s ever started, got %s", c.Name, o) + } + return nil +} + +// DoesntStart returns an error if the container was found to have started +func (o containerOutputList) DoesntStart(c containerTestConfig) error { + if idx := o.findIndex(c.Name, "Starting"); idx != -1 { + return fmt.Errorf("find container %s started, but didn't expect to, got %s", c.Name, o) + } + return nil +} + +// Exits returns an error if the container was not found to have exited +func (o containerOutputList) Exits(c containerTestConfig) error { + if idx := o.findIndex(c.Name, "Exiting"); idx == -1 { + return fmt.Errorf("couldn't find that container %s ever exited, got %s", c.Name, o) + } + return nil +} + +func (o containerOutputList) findIndex(name string, command string) int { + for _, v := range o { + if v.containerName == name && v.command == command { + return v.line + } + } + return -1 +} + +func parseOutput(dir string) containerOutputList { + contents, err := os.ReadFile(filepath.Join(dir, "output")) + framework.ExpectNoError(err, "reading output file") + + sc := bufio.NewScanner(bytes.NewReader(contents)) + var res containerOutputList + lineNo := 0 + for sc.Scan() { + lineNo++ + fields := strings.Fields(sc.Text()) + if len(fields) < 3 { + framework.ExpectNoError(fmt.Errorf("%v should have at least length 3", fields)) + } + res = append(res, containerOutput{ + line: lineNo, + timestamp: fields[0], + containerName: fields[1], + command: fields[2], + }) + } + return res +} + +func getContainerOrderingPod(podName string, hostDir string, containerConfigs ...containerTestConfig) *v1.Pod { + // all the pods share the given host directory + hostPathDirectory := v1.HostPathDirectory + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "shared", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: hostDir, + Type: &hostPathDirectory, + }, + }, + }, + }, + }, + } + + for _, cc := range containerConfigs { + if cc.Name == "" { + framework.Failf("expected container config to have a name, found %#v", cc) + } + c := v1.Container{ + Name: cc.Name, + Image: busyboxImage, + Command: cc.Command(), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("15Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("15Mi"), + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "shared", + MountPath: "/shared", + }, + }, + } + + switch cc.Type { + case containerTypeInit: + p.Spec.InitContainers = append(p.Spec.InitContainers, c) + case containerTypeRegular: + p.Spec.Containers = append(p.Spec.Containers, c) + default: + framework.Failf("expected container config to have a valid type, found %#v", cc) + } + } + + return p +}