From dec1c36125567c3709ec8122f0cb034ab05a6052 Mon Sep 17 00:00:00 2001 From: AxeZhan Date: Sat, 12 Aug 2023 20:45:48 +0800 Subject: [PATCH 1/4] add kubectl rollout e2e test --- test/e2e/kubectl/rollout.go | 145 ++++++++++++++++++ .../kubectl/labeled-deployment.yaml.in | 21 +++ 2 files changed, 166 insertions(+) create mode 100644 test/e2e/kubectl/rollout.go create mode 100644 test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in diff --git a/test/e2e/kubectl/rollout.go b/test/e2e/kubectl/rollout.go new file mode 100644 index 00000000000..cda0d4fc602 --- /dev/null +++ b/test/e2e/kubectl/rollout.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// OWNER = sig/cli + +package kubectl + +import ( + "context" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + commonutils "k8s.io/kubernetes/test/e2e/common" + "k8s.io/kubernetes/test/e2e/framework" + e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + admissionapi "k8s.io/pod-security-admission/api" +) + +const ( + labeledDeploymentFilename = "labeled-deployment.yaml.in" +) + +var _ = SIGDescribe("Kubectl rollout", func() { + defer ginkgo.GinkgoRecover() + f := framework.NewDefaultFramework("kubectl-rollout") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + var c clientset.Interface + var ns string + ginkgo.BeforeEach(func() { + c = f.ClientSet + ns = f.Namespace.Name + }) + + ginkgo.Describe("undo", func() { + deploymentYaml := commonutils.SubstituteImageName(string(readTestFileOrDie(labeledDeploymentFilename))) + ginkgo.AfterEach(func() { + cleanupKubectlInputs(deploymentYaml, ns, "labels_foo=labels_bar") + }) + ginkgo.It("undo should rollback and update deployment env", func(ctx context.Context) { + var err error + // create deployment + e2ekubectl.RunKubectlOrDieInput(ns, deploymentYaml, "apply", "-f", "-") + + replica := int32(1) + d := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pause-deployment", + Namespace: ns, + }, + Spec: appsv1.DeploymentSpec{Replicas: &replica}, + } + + if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { + framework.Failf("created deployment not ready") + } + + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + framework.Failf("get deployment failed") + } + + origEnv := d.Spec.Template.Spec.Containers[0].Env + origLabels := d.Spec.Template.Labels + origAnnotations := d.Spec.Template.Annotations + + for _, env := range origEnv { + if env.Name == "foo" && env.Value == "bar" { + framework.Failf("labeled deployment should not have an env named foo and valued bar at the beginning") + } + } + + // do a small update + if _, err = e2ekubectl.RunKubectl(ns, "set", "env", "deployment/pause-deployment", "foo=bar"); err != nil { + framework.Failf("kubectl failed set env for deployment") + } + // wait for env to be set + if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { + framework.Failf("update deployment failed") + } + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + framework.Failf("get deployment failed") + } + envs := d.Spec.Template.Spec.Containers[0].Env + + envUpdated := false + for _, env := range envs { + if env.Name == "foo" && env.Value == "bar" { + envUpdated = true + break + } + } + if !envUpdated { + framework.Failf("update deployment's env failed") + } + + // rollback + if _, err = e2ekubectl.RunKubectl(ns, "rollout", "undo", "deployment/pause-deployment"); err != nil { + framework.Failf("kubectl failed to rollback deployment") + } + // wait for rollback finished + if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { + framework.Failf("rollback deployment failed") + } + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + framework.Failf("get deployment failed") + } + + rollbackedEnv := d.Spec.Template.Spec.Containers[0].Env + rollbackedLabels := d.Spec.Template.Labels + rollbackedAnnotations := d.Spec.Template.Annotations + + if diff := cmp.Diff(origEnv, rollbackedEnv, cmpopts.SortSlices(func(a, b v1.EnvVar) bool { + return a.Name < b.Name + })); diff != "" { + framework.Failf("inconsistent env after rolled back: %s", diff) + } + if diff := cmp.Diff(origLabels, rollbackedLabels); diff != "" { + framework.Failf("inconsistent labels after rolled back: %s", diff) + } + if diff := cmp.Diff(origAnnotations, rollbackedAnnotations); diff != "" { + framework.Failf("inconsistent annotations after rolled back: %s", diff) + } + }) + }) +}) diff --git a/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in b/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in new file mode 100644 index 00000000000..64bf36eee90 --- /dev/null +++ b/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pause-deployment +spec: + replicas: 1 + selector: + matchLabels: + labels_foo: labels_bar + template: + metadata: + labels: + labels_foo: labels_bar + annotations: + annotations_foo: annotations_bar + spec: + containers: + - name: pause + image: {{.PauseImage}} + ports: + - containerPort: 80 From ff10df107a498a5b09a877ff4c8438d51f59cd5b Mon Sep 17 00:00:00 2001 From: AxeZhan Date: Fri, 25 Aug 2023 21:52:33 +0800 Subject: [PATCH 2/4] remove fake deployment --- test/e2e/kubectl/rollout.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/test/e2e/kubectl/rollout.go b/test/e2e/kubectl/rollout.go index cda0d4fc602..0c506b8281e 100644 --- a/test/e2e/kubectl/rollout.go +++ b/test/e2e/kubectl/rollout.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -62,19 +63,11 @@ var _ = SIGDescribe("Kubectl rollout", func() { // create deployment e2ekubectl.RunKubectlOrDieInput(ns, deploymentYaml, "apply", "-f", "-") - replica := int32(1) - d := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pause-deployment", - Namespace: ns, - }, - Spec: appsv1.DeploymentSpec{Replicas: &replica}, - } - - if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { + if err = e2edeployment.WaitForDeploymentRevisionAndImage(c, ns, "pause-deployment", "1", imageutils.GetE2EImage(imageutils.Pause)); err != nil { framework.Failf("created deployment not ready") } + var d *appsv1.Deployment if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { framework.Failf("get deployment failed") } From ff3cb43c3002dc1d5ec014017d4e9f0cf79a63aa Mon Sep 17 00:00:00 2001 From: AxeZhan Date: Mon, 4 Sep 2023 18:34:28 +0800 Subject: [PATCH 3/4] remove labeld-deployment.yaml --- test/e2e/kubectl/rollout.go | 20 +++++++----------- .../kubectl/httpd-deployment1.yaml.in | 2 ++ .../kubectl/labeled-deployment.yaml.in | 21 ------------------- 3 files changed, 10 insertions(+), 33 deletions(-) delete mode 100644 test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in diff --git a/test/e2e/kubectl/rollout.go b/test/e2e/kubectl/rollout.go index 0c506b8281e..f51ea382018 100644 --- a/test/e2e/kubectl/rollout.go +++ b/test/e2e/kubectl/rollout.go @@ -37,10 +37,6 @@ import ( admissionapi "k8s.io/pod-security-admission/api" ) -const ( - labeledDeploymentFilename = "labeled-deployment.yaml.in" -) - var _ = SIGDescribe("Kubectl rollout", func() { defer ginkgo.GinkgoRecover() f := framework.NewDefaultFramework("kubectl-rollout") @@ -54,21 +50,21 @@ var _ = SIGDescribe("Kubectl rollout", func() { }) ginkgo.Describe("undo", func() { - deploymentYaml := commonutils.SubstituteImageName(string(readTestFileOrDie(labeledDeploymentFilename))) + deploymentYaml := commonutils.SubstituteImageName(string(readTestFileOrDie(httpdDeployment1Filename))) ginkgo.AfterEach(func() { - cleanupKubectlInputs(deploymentYaml, ns, "labels_foo=labels_bar") + cleanupKubectlInputs(deploymentYaml, ns, "app=httpd") }) ginkgo.It("undo should rollback and update deployment env", func(ctx context.Context) { var err error // create deployment e2ekubectl.RunKubectlOrDieInput(ns, deploymentYaml, "apply", "-f", "-") - if err = e2edeployment.WaitForDeploymentRevisionAndImage(c, ns, "pause-deployment", "1", imageutils.GetE2EImage(imageutils.Pause)); err != nil { + if err = e2edeployment.WaitForDeploymentRevisionAndImage(c, ns, "httpd-deployment", "1", imageutils.GetE2EImage(imageutils.HttpdNew)); err != nil { framework.Failf("created deployment not ready") } var d *appsv1.Deployment - if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "httpd-deployment", metav1.GetOptions{}); err != nil { framework.Failf("get deployment failed") } @@ -83,14 +79,14 @@ var _ = SIGDescribe("Kubectl rollout", func() { } // do a small update - if _, err = e2ekubectl.RunKubectl(ns, "set", "env", "deployment/pause-deployment", "foo=bar"); err != nil { + if _, err = e2ekubectl.RunKubectl(ns, "set", "env", "deployment/httpd-deployment", "foo=bar"); err != nil { framework.Failf("kubectl failed set env for deployment") } // wait for env to be set if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { framework.Failf("update deployment failed") } - if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "httpd-deployment", metav1.GetOptions{}); err != nil { framework.Failf("get deployment failed") } envs := d.Spec.Template.Spec.Containers[0].Env @@ -107,14 +103,14 @@ var _ = SIGDescribe("Kubectl rollout", func() { } // rollback - if _, err = e2ekubectl.RunKubectl(ns, "rollout", "undo", "deployment/pause-deployment"); err != nil { + if _, err = e2ekubectl.RunKubectl(ns, "rollout", "undo", "deployment/httpd-deployment"); err != nil { framework.Failf("kubectl failed to rollback deployment") } // wait for rollback finished if err = e2edeployment.WaitForDeploymentComplete(c, d); err != nil { framework.Failf("rollback deployment failed") } - if d, err = c.AppsV1().Deployments(ns).Get(ctx, "pause-deployment", metav1.GetOptions{}); err != nil { + if d, err = c.AppsV1().Deployments(ns).Get(ctx, "httpd-deployment", metav1.GetOptions{}); err != nil { framework.Failf("get deployment failed") } diff --git a/test/e2e/testing-manifests/kubectl/httpd-deployment1.yaml.in b/test/e2e/testing-manifests/kubectl/httpd-deployment1.yaml.in index 3b82a2fd2f0..72164958b89 100644 --- a/test/e2e/testing-manifests/kubectl/httpd-deployment1.yaml.in +++ b/test/e2e/testing-manifests/kubectl/httpd-deployment1.yaml.in @@ -11,6 +11,8 @@ spec: metadata: labels: app: httpd + annotations: + annotations_app: annotations_httpd spec: containers: - name: httpd diff --git a/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in b/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in deleted file mode 100644 index 64bf36eee90..00000000000 --- a/test/e2e/testing-manifests/kubectl/labeled-deployment.yaml.in +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pause-deployment -spec: - replicas: 1 - selector: - matchLabels: - labels_foo: labels_bar - template: - metadata: - labels: - labels_foo: labels_bar - annotations: - annotations_foo: annotations_bar - spec: - containers: - - name: pause - image: {{.PauseImage}} - ports: - - containerPort: 80 From 059e8353b76cd7f8f0702aa5e2b0e577c79dd0ac Mon Sep 17 00:00:00 2001 From: AxeZhan Date: Tue, 5 Sep 2023 19:23:01 +0800 Subject: [PATCH 4/4] check labels/annotations in the beginning --- test/e2e/kubectl/rollout.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/kubectl/rollout.go b/test/e2e/kubectl/rollout.go index f51ea382018..fe4c296b0e0 100644 --- a/test/e2e/kubectl/rollout.go +++ b/test/e2e/kubectl/rollout.go @@ -69,15 +69,22 @@ var _ = SIGDescribe("Kubectl rollout", func() { } origEnv := d.Spec.Template.Spec.Containers[0].Env - origLabels := d.Spec.Template.Labels - origAnnotations := d.Spec.Template.Annotations - for _, env := range origEnv { if env.Name == "foo" && env.Value == "bar" { framework.Failf("labeled deployment should not have an env named foo and valued bar at the beginning") } } + origLabels := d.Spec.Template.Labels + if len(origLabels) == 0 { + framework.Failf("original labels should not be empty in kubectl rollout test") + } + + origAnnotations := d.Spec.Template.Annotations + if len(origAnnotations) == 0 { + framework.Failf("original annotations should not be empty in kubectl rollout test") + } + // do a small update if _, err = e2ekubectl.RunKubectl(ns, "set", "env", "deployment/httpd-deployment", "foo=bar"); err != nil { framework.Failf("kubectl failed set env for deployment")