From 7eb9b81d67076b5f8855560bc081d1d36ba56bfb Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Fri, 24 Feb 2017 16:44:21 -0800 Subject: [PATCH] Updates test/e2e/addon_update.go to match addon-manager's new behavior --- test/e2e/addon_update.go | 254 +++++++++++++++++++------------------ test/e2e/framework/util.go | 58 +++++++-- 2 files changed, 179 insertions(+), 133 deletions(-) diff --git a/test/e2e/addon_update.go b/test/e2e/addon_update.go index 666a2217607..74cd41ad58b 100644 --- a/test/e2e/addon_update.go +++ b/test/e2e/addon_update.go @@ -26,6 +26,7 @@ import ( "golang.org/x/crypto/ssh" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" @@ -35,126 +36,156 @@ import ( // TODO: it would probably be slightly better to build up the objects // in the code and then serialize to yaml. -var addon_controller_v1 = ` +var reconcile_addon_controller = ` apiVersion: v1 kind: ReplicationController metadata: - name: addon-test-v1 + name: addon-reconcile-test namespace: %s labels: - k8s-app: addon-test - version: v1 + k8s-app: addon-reconcile-test kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: replicas: 2 selector: - k8s-app: addon-test - version: v1 + k8s-app: addon-reconcile-test template: metadata: labels: - k8s-app: addon-test - version: v1 - kubernetes.io/cluster-service: "true" + k8s-app: addon-reconcile-test spec: containers: - image: gcr.io/google_containers/serve_hostname:v1.4 - name: addon-test + name: addon-reconcile-test ports: - containerPort: 9376 protocol: TCP ` -var addon_controller_v2 = ` +// Should update "reconcile" class addon. +var reconcile_addon_controller_updated = ` apiVersion: v1 kind: ReplicationController metadata: - name: addon-test-v2 + name: addon-reconcile-test namespace: %s labels: - k8s-app: addon-test - version: v2 + k8s-app: addon-reconcile-test kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + newLabel: addon-reconcile-test spec: replicas: 2 selector: - k8s-app: addon-test - version: v2 + k8s-app: addon-reconcile-test template: metadata: labels: - k8s-app: addon-test - version: v2 - kubernetes.io/cluster-service: "true" + k8s-app: addon-reconcile-test spec: containers: - image: gcr.io/google_containers/serve_hostname:v1.4 - name: addon-test + name: addon-reconcile-test ports: - containerPort: 9376 protocol: TCP ` -var addon_service_v1 = ` +var ensure_exists_addon_service = ` apiVersion: v1 kind: Service metadata: - name: addon-test + name: addon-ensure-exists-test namespace: %s labels: - k8s-app: addon-test - kubernetes.io/cluster-service: "true" - kubernetes.io/name: addon-test + k8s-app: addon-ensure-exists-test + addonmanager.kubernetes.io/mode: EnsureExists spec: ports: - port: 9376 protocol: TCP targetPort: 9376 selector: - k8s-app: addon-test + k8s-app: addon-ensure-exists-test ` -var addon_service_v2 = ` +// Should create but don't update "ensure exist" class addon. +var ensure_exists_addon_service_updated = ` apiVersion: v1 kind: Service metadata: - name: addon-test-updated + name: addon-ensure-exists-test namespace: %s labels: - k8s-app: addon-test - kubernetes.io/cluster-service: "true" - kubernetes.io/name: addon-test - newLabel: newValue + k8s-app: addon-ensure-exists-test + addonmanager.kubernetes.io/mode: EnsureExists + newLabel: addon-ensure-exists-test spec: ports: - port: 9376 protocol: TCP targetPort: 9376 selector: - k8s-app: addon-test + k8s-app: addon-ensure-exists-test ` -// Wrong label case -var invalid_addon_controller_v1 = ` +var deprecated_label_addon_service = ` +apiVersion: v1 +kind: Service +metadata: + name: addon-deprecated-label-test + namespace: %s + labels: + k8s-app: addon-deprecated-label-test + kubernetes.io/cluster-service: "true" +spec: + ports: + - port: 9376 + protocol: TCP + targetPort: 9376 + selector: + k8s-app: addon-deprecated-label-test +` + +// Should update addon with label "kubernetes.io/cluster-service=true". +var deprecated_label_addon_service_updated = ` +apiVersion: v1 +kind: Service +metadata: + name: addon-deprecated-label-test + namespace: %s + labels: + k8s-app: addon-deprecated-label-test + kubernetes.io/cluster-service: "true" + newLabel: addon-deprecated-label-test +spec: + ports: + - port: 9376 + protocol: TCP + targetPort: 9376 + selector: + k8s-app: addon-deprecated-label-test +` + +// Should not create addon without valid label. +var invalid_addon_controller = ` apiVersion: v1 kind: ReplicationController metadata: - name: invalid-addon-test-v1 + name: invalid-addon-test namespace: %s labels: k8s-app: invalid-addon-test - version: v1 + addonmanager.kubernetes.io/mode: NotMatch spec: replicas: 2 selector: k8s-app: invalid-addon-test - version: v1 template: metadata: labels: k8s-app: invalid-addon-test - version: v1 - kubernetes.io/cluster-service: "true" spec: containers: - image: gcr.io/google_containers/serve_hostname:v1.4 @@ -164,49 +195,10 @@ spec: protocol: TCP ` -// Wrong label case -var invalid_addon_service_v1 = ` -apiVersion: v1 -kind: Service -metadata: - name: ivalid-addon-test - namespace: %s - labels: - k8s-app: invalid-addon-test - kubernetes.io/name: invalid-addon-test -spec: - ports: - - port: 9377 - protocol: TCP - targetPort: 9376 - selector: - k8s-app: invalid-addon-test -` - -// Wrong namespace case -var invalid_addon_service_v2 = ` -apiVersion: v1 -kind: Service -metadata: - name: ivalid-addon-test-v2 - namespace: %s - labels: - k8s-app: invalid-addon-test-v2 - kubernetes.io/cluster-service: "true" -spec: - ports: - - port: 9377 - protocol: TCP - targetPort: 9376 - selector: - k8s-app: invalid-addon-test -` - const ( addonTestPollInterval = 3 * time.Second addonTestPollTimeout = 5 * time.Minute - defaultNsName = metav1.NamespaceDefault - addonNsName = "kube-system" + addonNsName = metav1.NamespaceSystem ) type stringPair struct { @@ -257,23 +249,23 @@ var _ = framework.KubeDescribe("Addon update", func() { defer sshExec(sshClient, fmt.Sprintf("rm -rf %s", temporaryRemotePathPrefix)) // ignore the result in cleanup sshExecAndVerify(sshClient, fmt.Sprintf("mkdir -p %s", temporaryRemotePath)) - rcv1 := "addon-controller-v1.yaml" - rcv2 := "addon-controller-v2.yaml" - rcInvalid := "invalid-addon-controller-v1.yaml" + rcAddonReconcile := "addon-reconcile-controller.yaml" + rcAddonReconcileUpdated := "addon-reconcile-controller-Updated.yaml" + rcInvalid := "invalid-addon-controller.yaml" - svcv1 := "addon-service-v1.yaml" - svcv2 := "addon-service-v2.yaml" - svcInvalid := "invalid-addon-service-v1.yaml" - svcInvalidv2 := "invalid-addon-service-v2.yaml" + svcAddonDeprecatedLabel := "addon-deprecated-label-service.yaml" + svcAddonDeprecatedLabelUpdated := "addon-deprecated-label-service-updated.yaml" + svcAddonEnsureExists := "addon-ensure-exists-service.yaml" + svcAddonEnsureExistsUpdated := "addon-ensure-exists-service-updated.yaml" var remoteFiles []stringPair = []stringPair{ - {fmt.Sprintf(addon_controller_v1, addonNsName), rcv1}, - {fmt.Sprintf(addon_controller_v2, addonNsName), rcv2}, - {fmt.Sprintf(addon_service_v1, addonNsName), svcv1}, - {fmt.Sprintf(addon_service_v2, addonNsName), svcv2}, - {fmt.Sprintf(invalid_addon_controller_v1, addonNsName), rcInvalid}, - {fmt.Sprintf(invalid_addon_service_v1, addonNsName), svcInvalid}, - {fmt.Sprintf(invalid_addon_service_v2, defaultNsName), svcInvalidv2}, + {fmt.Sprintf(reconcile_addon_controller, addonNsName), rcAddonReconcile}, + {fmt.Sprintf(reconcile_addon_controller_updated, addonNsName), rcAddonReconcileUpdated}, + {fmt.Sprintf(deprecated_label_addon_service, addonNsName), svcAddonDeprecatedLabel}, + {fmt.Sprintf(deprecated_label_addon_service_updated, addonNsName), svcAddonDeprecatedLabelUpdated}, + {fmt.Sprintf(ensure_exists_addon_service, addonNsName), svcAddonEnsureExists}, + {fmt.Sprintf(ensure_exists_addon_service_updated, addonNsName), svcAddonEnsureExistsUpdated}, + {fmt.Sprintf(invalid_addon_controller, addonNsName), rcInvalid}, } for _, p := range remoteFiles { @@ -292,51 +284,54 @@ var _ = framework.KubeDescribe("Addon update", func() { defer sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix)) // ignore result in cleanup sshExecAndVerify(sshClient, fmt.Sprintf("sudo mkdir -p %s", destinationDir)) - By("copy invalid manifests to the destination dir (without kubernetes.io/cluster-service label)") + By("copy invalid manifests to the destination dir") sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcInvalid, destinationDir, rcInvalid)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcInvalid, destinationDir, svcInvalid)) // we will verify at the end of the test that the objects weren't created from the invalid manifests By("copy new manifests") - sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv1, destinationDir, rcv1)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv1, destinationDir, svcv1)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcAddonReconcile, destinationDir, rcAddonReconcile)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonDeprecatedLabel, destinationDir, svcAddonDeprecatedLabel)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonEnsureExists, destinationDir, svcAddonEnsureExists)) + // Delete the "ensure exist class" addon at the end. + defer func() { + framework.Logf("Cleaning up ensure exist class addon.") + Expect(f.ClientSet.Core().Services(addonNsName).Delete("addon-ensure-exists-test", nil)).NotTo(HaveOccurred()) + }() - waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-test", true) - waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-test-v1", true) + waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-reconcile-test", true) + waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-deprecated-label-test", true) + waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-ensure-exists-test", true) + // Replace the manifests with new contents. By("update manifests") - sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcv2, destinationDir, rcv2)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcv2, destinationDir, svcv2)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv1)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv1)) - /** - * Note that we have a small race condition here - the kube-addon-updater - * May notice that a new rc/service file appeared, while the old one will still be there. - * But it is ok - as long as we don't have rolling update, the result will be the same - */ + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcAddonReconcileUpdated, destinationDir, rcAddonReconcile)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonDeprecatedLabelUpdated, destinationDir, svcAddonDeprecatedLabel)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonEnsureExistsUpdated, destinationDir, svcAddonEnsureExists)) - waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-test-updated", true) - waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-test-v2", true) - - waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-test", false) - waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-test-v1", false) + // Wait for updated addons to have the new added label. + reconcileSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-reconcile-test"})) + waitForReplicationControllerwithSelectorInAddonTest(f.ClientSet, addonNsName, true, reconcileSelector) + deprecatedLabelSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-deprecated-label-test"})) + waitForServicewithSelectorInAddonTest(f.ClientSet, addonNsName, true, deprecatedLabelSelector) + // "Ensure exist class" addon should not be updated. + ensureExistSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-ensure-exists-test"})) + waitForServicewithSelectorInAddonTest(f.ClientSet, addonNsName, false, ensureExistSelector) By("remove manifests") - sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcv2)) - sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcv2)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcAddonReconcile)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcAddonDeprecatedLabel)) + sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcAddonEnsureExists)) - waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-test-updated", false) - waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-test-v2", false) + waitForReplicationControllerInAddonTest(f.ClientSet, addonNsName, "addon-reconcile-test", false) + waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-deprecated-label-test", false) + // "Ensure exist class" addon will not be deleted when manifest is removed. + waitForServiceInAddonTest(f.ClientSet, addonNsName, "addon-ensure-exists-test", true) - By("verify invalid API addons weren't created") - _, err = f.ClientSet.Core().ReplicationControllers(addonNsName).Get("invalid-addon-test-v1", metav1.GetOptions{}) - Expect(err).To(HaveOccurred()) - _, err = f.ClientSet.Core().Services(addonNsName).Get("ivalid-addon-test", metav1.GetOptions{}) - Expect(err).To(HaveOccurred()) - _, err = f.ClientSet.Core().Services(defaultNsName).Get("ivalid-addon-test-v2", metav1.GetOptions{}) + By("verify invalid addons weren't created") + _, err = f.ClientSet.Core().ReplicationControllers(addonNsName).Get("invalid-addon-test", metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - // invalid addons will be deleted by the deferred function + // Invalid addon manifests and the "ensure exist class" addon will be deleted by the deferred function. }) }) @@ -348,6 +343,15 @@ func waitForReplicationControllerInAddonTest(c clientset.Interface, addonNamespa framework.ExpectNoError(framework.WaitForReplicationController(c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout)) } +func waitForServicewithSelectorInAddonTest(c clientset.Interface, addonNamespace string, exist bool, selector labels.Selector) { + framework.ExpectNoError(framework.WaitForServiceWithSelector(c, addonNamespace, selector, exist, addonTestPollInterval, addonTestPollTimeout)) +} + +func waitForReplicationControllerwithSelectorInAddonTest(c clientset.Interface, addonNamespace string, exist bool, selector labels.Selector) { + framework.ExpectNoError(framework.WaitForReplicationControllerwithSelector(c, addonNamespace, selector, exist, addonTestPollInterval, + addonTestPollTimeout)) +} + // TODO use the framework.SSH code, either adding an SCP to it or copying files // differently. func getMasterSSHClient() (*ssh.Client, error) { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 42384bee790..884c18647dd 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1456,17 +1456,11 @@ func WaitForService(c clientset.Interface, namespace, name string, exist bool, i _, err := c.Core().Services(namespace).Get(name, metav1.GetOptions{}) switch { case err == nil: - if !exist { - return false, nil - } Logf("Service %s in namespace %s found.", name, namespace) - return true, nil + return exist, nil case apierrs.IsNotFound(err): - if exist { - return false, nil - } Logf("Service %s in namespace %s disappeared.", name, namespace) - return true, nil + return !exist, nil default: Logf("Get service %s in namespace %s failed: %v", name, namespace, err) return false, nil @@ -1479,6 +1473,30 @@ func WaitForService(c clientset.Interface, namespace, name string, exist bool, i return nil } +// WaitForServiceWithSelector waits until any service with given selector appears (exist == true), or disappears (exist == false) +func WaitForServiceWithSelector(c clientset.Interface, namespace string, selector labels.Selector, exist bool, interval, + timeout time.Duration) error { + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + services, err := c.Core().Services(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) + switch { + case len(services.Items) != 0: + Logf("Service with %s in namespace %s found.", selector.String(), namespace) + return exist, nil + case len(services.Items) == 0: + Logf("Service with %s in namespace %s disappeared.", selector.String(), namespace) + return !exist, nil + default: + Logf("List service with %s in namespace %s failed: %v", selector.String(), namespace, err) + return false, nil + } + }) + if err != nil { + stateMsg := map[bool]string{true: "to appear", false: "to disappear"} + return fmt.Errorf("error waiting for service with %s in namespace %s %s: %v", selector.String(), namespace, stateMsg[exist], err) + } + return nil +} + //WaitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum. func WaitForServiceEndpointsNum(c clientset.Interface, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error { return wait.Poll(interval, timeout, func() (bool, error) { @@ -1524,6 +1542,30 @@ func WaitForReplicationController(c clientset.Interface, namespace, name string, return nil } +// WaitForReplicationControllerwithSelector waits until any RC with given selector appears (exist == true), or disappears (exist == false) +func WaitForReplicationControllerwithSelector(c clientset.Interface, namespace string, selector labels.Selector, exist bool, interval, + timeout time.Duration) error { + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + rcs, err := c.Core().ReplicationControllers(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) + switch { + case len(rcs.Items) != 0: + Logf("ReplicationController with %s in namespace %s found.", selector.String(), namespace) + return exist, nil + case len(rcs.Items) == 0: + Logf("ReplicationController with %s in namespace %s disappeared.", selector.String(), namespace) + return !exist, nil + default: + Logf("List ReplicationController with %s in namespace %s failed: %v", selector.String(), namespace, err) + return false, nil + } + }) + if err != nil { + stateMsg := map[bool]string{true: "to appear", false: "to disappear"} + return fmt.Errorf("error waiting for ReplicationControllers with %s in namespace %s %s: %v", selector.String(), namespace, stateMsg[exist], err) + } + return nil +} + func WaitForEndpoint(c clientset.Interface, ns, name string) error { for t := time.Now(); time.Since(t) < EndpointRegisterTimeout; time.Sleep(Poll) { endpoint, err := c.Core().Endpoints(ns).Get(name, metav1.GetOptions{})