From 3a5dda78afd0f548a362ed5d14a948907c0968ef Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 29 Apr 2025 20:43:55 +0000 Subject: [PATCH] e2e network: blue green deployments Use Pod Readiness Gates and Services to implement blue green deployment strategies by setting the corresponding readines gate to influence the service traffic. Change-Id: I23e3b9a0440014f48a4612685055565fd8dff5ec --- test/e2e/network/pod_lifecycle.go | 237 ++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) diff --git a/test/e2e/network/pod_lifecycle.go b/test/e2e/network/pod_lifecycle.go index 837caa7b2d9..588c65bd629 100644 --- a/test/e2e/network/pod_lifecycle.go +++ b/test/e2e/network/pod_lifecycle.go @@ -21,13 +21,20 @@ import ( "fmt" "net" "strconv" + "strings" "time" "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + e2eservice "k8s.io/kubernetes/test/e2e/framework/service" "k8s.io/kubernetes/test/e2e/network/common" admissionapi "k8s.io/pod-security-admission/api" ) @@ -98,4 +105,234 @@ var _ = common.SIGDescribe("Connectivity Pod Lifecycle", func() { execHostnameTest(*pausePod, podIPAddress, webserverPod.Name) }) + ginkgo.It("should be able to have zero downtime on a Blue Green deployment using Services and Readiness Gates", func(ctx context.Context) { + readinessGate := "k8s.io/blue-green" + patchStatusFmt := `{"status":{"conditions":[{"type":%q, "status":%q}]}}` + + serviceName := "blue-green-svc" + blueGreenJig := e2eservice.NewTestJig(cs, ns, serviceName) + ginkgo.By("creating a service " + serviceName + " with type=ClusterIP in " + ns) + blueGreenService, err := blueGreenJig.CreateTCPService(ctx, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeClusterIP + svc.Spec.Ports = []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP, TargetPort: intstr.FromInt32(80)}, + } + }) + framework.ExpectNoError(err) + + ginkgo.By("Creating 2 webserver pod (green and blue) able to serve traffic during the grace period of 30 seconds") + gracePeriod := int64(30) + bluePod := e2epod.NewAgnhostPod(ns, "blue-pod", nil, nil, nil, "netexec", "--http-port=80", fmt.Sprintf("--delay-shutdown=%d", gracePeriod)) + bluePod.Labels = blueGreenService.Labels + bluePod.Spec.ReadinessGates = []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType(readinessGate)}, + } + podClient.Create(ctx, bluePod) + err = e2epod.WaitForPodNameRunningInNamespace(ctx, cs, bluePod.Name, ns) + if err != nil { + framework.Failf("waiting for pod %s : %v", bluePod.Name, err) + } + if podClient.PodIsReady(ctx, bluePod.Name) { + framework.Failf("Expect pod(%s/%s)'s Ready condition to be false initially.", ns, bluePod.Name) + } + + greenPod := e2epod.NewAgnhostPod(ns, "green-pod", nil, nil, nil, "netexec", "--http-port=80", fmt.Sprintf("--delay-shutdown=%d", gracePeriod)) + greenPod.Labels = blueGreenService.Labels + greenPod.Spec.ReadinessGates = []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType(readinessGate)}, + } + podClient.Create(ctx, greenPod) + err = e2epod.WaitForPodNameRunningInNamespace(ctx, cs, greenPod.Name, ns) + if err != nil { + framework.Failf("waiting for pod %s : %v", greenPod.Name, err) + } + if podClient.PodIsReady(ctx, greenPod.Name) { + framework.Failf("Expect pod(%s/%s)'s Ready condition to be false initially.", ns, greenPod.Name) + } + + ginkgo.By("Creating 1 client pod that will try to connect to the blue-green-svc") + clientPod := e2epod.NewAgnhostPod(ns, "client-pod-1", nil, nil, nil) + clientPod.Spec.TerminationGracePeriodSeconds = &gracePeriod + clientPod = podClient.CreateSync(ctx, clientPod) + + ginkgo.By(fmt.Sprintf("patching blue pod status with condition %q to true", readinessGate)) + _, err = podClient.Patch(ctx, bluePod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "True")), metav1.PatchOptions{}, "status") + if err != nil { + framework.Failf("failed to patch %s pod condition: %v", bluePod.Name, err) + } + + // Expect EndpointSlice resource to have the blue pod ready to serve traffic + if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) { + endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name, + }) + if err != nil { + return false, err + } + for _, slice := range endpointSliceList.Items { + for _, ep := range slice.Endpoints { + if ep.TargetRef != nil && + ep.TargetRef.Name == bluePod.Name && + ep.TargetRef.Namespace == bluePod.Namespace && + ep.Conditions.Ready != nil && *ep.Conditions.Ready { + return true, nil + } + } + } + return false, nil + }); err != nil { + framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err) + } + + ginkgo.By("Try to connect to the blue pod through the service") + scvAddress := net.JoinHostPort(blueGreenService.Spec.ClusterIP, strconv.Itoa(80)) + // assert 5 times that we can connect only to the blue pod + for i := 0; i < 5; i++ { + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) { + cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress) + stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd) + if err != nil { + framework.Logf("expected error when trying to connect to cluster IP : %v", err) + return false, nil + } + if strings.TrimSpace(stdout) == "" { + framework.Logf("got empty stdout, retry until timeout") + return false, nil + } + // Ensure we're comparing hostnames and not FQDNs + targetHostname := strings.Split(bluePod.Name, ".")[0] + hostname := strings.TrimSpace(strings.Split(stdout, ".")[0]) + if hostname != targetHostname { + return false, fmt.Errorf("expecting hostname %s got %s", targetHostname, hostname) + } + return true, nil + }) + if err != nil { + framework.Failf("can not connect to pod %s on address %s : %v", bluePod.Name, scvAddress, err) + } + } + + // Switch from blue to green + ginkgo.By(fmt.Sprintf("patching green pod status with condition %q to true", readinessGate)) + _, err = podClient.Patch(ctx, greenPod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "True")), metav1.PatchOptions{}, "status") + if err != nil { + framework.Failf("failed to patch %s pod condition: %v", greenPod.Name, err) + } + + // Expect EndpointSlice resource to have the green pod ready to serve traffic + if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) { + endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name, + }) + if err != nil { + return false, err + } + for _, slice := range endpointSliceList.Items { + for _, ep := range slice.Endpoints { + if ep.TargetRef != nil && + ep.TargetRef.Name == greenPod.Name && + ep.TargetRef.Namespace == greenPod.Namespace && + ep.Conditions.Ready != nil && *ep.Conditions.Ready { + return true, nil + } + } + } + return false, nil + }); err != nil { + framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err) + } + + ginkgo.By(fmt.Sprintf("patching blue pod status with condition %q to false", readinessGate)) + _, err = podClient.Patch(ctx, bluePod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "False")), metav1.PatchOptions{}, "status") + if err != nil { + framework.Failf("failed to patch %s pod condition: %v", bluePod.Name, err) + } + + // Expect EndpointSlice resource to have the blue pod NOT ready to serve traffic + if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) { + endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name, + }) + if err != nil { + return false, err + } + for _, slice := range endpointSliceList.Items { + for _, ep := range slice.Endpoints { + if ep.TargetRef != nil && + ep.TargetRef.Name == bluePod.Name && + ep.TargetRef.Namespace == bluePod.Namespace && + ep.Conditions.Ready != nil && !*ep.Conditions.Ready { + return true, nil + } + } + } + return false, nil + }); err != nil { + framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err) + } + + // We have checked the endpoint slices reflect the desired state: + // bluePod not ready and greenPod ready, but we need to remember kubernetes + // is a distributed system eventually consistent, so there is a propagation + // delay until this information is present on the nodes and a programming delay + // until the corresponding node components program the information on the dataplane. + err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) { + cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress) + stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd) + if err != nil { + framework.Logf("expected error when trying to connect to cluster IP : %v", err) + return false, nil + } + if strings.TrimSpace(stdout) == "" { + framework.Logf("got empty stdout, retry until timeout") + return false, nil + } + // Ensure we're comparing hostnames and not FQDNs + targetHostname := strings.Split(greenPod.Name, ".")[0] + hostname := strings.TrimSpace(strings.Split(stdout, ".")[0]) + if hostname != targetHostname { + framework.Logf("expecting hostname %s got %s", targetHostname, hostname) + return false, nil + } + return true, nil + }) + if err != nil { + framework.Failf("can not connect to pod %s on address %s : %v", greenPod.Name, scvAddress, err) + } + + ginkgo.By("Try to connect to the green pod through the service") + // assert 5 times that we can connect only to the green pod + for i := 0; i < 5; i++ { + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) { + cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress) + stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd) + if err != nil { + framework.Logf("expected error when trying to connect to cluster IP : %v", err) + return false, nil + } + if strings.TrimSpace(stdout) == "" { + framework.Logf("got empty stdout, retry until timeout") + return false, nil + } + // Ensure we're comparing hostnames and not FQDNs + targetHostname := strings.Split(greenPod.Name, ".")[0] + hostname := strings.TrimSpace(strings.Split(stdout, ".")[0]) + // At this point we should only receive traffic from the green Pod. + if hostname != targetHostname { + return false, fmt.Errorf("expecting hostname %s got %s", targetHostname, hostname) + } + return true, nil + }) + if err != nil { + framework.Failf("can not connect to pod %s on address %s : %v", greenPod.Name, scvAddress, err) + } + } + + // TODO there can be multiple combinations like: + // test zero downtime deleting the blue pod instead setting the readiness to false + // test roll back setting back the readiness to true on the blue pod + // ... + + }) + })