Merge pull request #39278 from jszczepkowski/ha-e2e-zones

Automatic merge from submit-queue

Added e2e test for HA master that creates multizone workers.

hack/e2e-internal/e2e-grow-cluster.sh

@@ -19,8 +19,17 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 if [[ ! -z "${1:-}" ]]; then
   export KUBE_GCE_ZONE="${1}"
 fi
-export KUBE_REPLICATE_EXISTING_MASTER=true
+if [[ ! -z "${2:-}" ]]; then
-export MULTIZONE=true
+  export MULTIZONE="${2}"
+fi
+if [[ ! -z "${3:-}" ]]; then
+  export KUBE_REPLICATE_EXISTING_MASTER="${3}"
+fi
+if [[ ! -z "${4:-}" ]]; then
+  export KUBE_USE_EXISTING_MASTER="${4}"
+fi
+if [[ -z "${NUM_NODES:-}" ]]; then
+  export NUM_NODES=3
+fi
 
 source "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"
-

hack/e2e-internal/e2e-shrink-cluster.sh

@@ -22,7 +22,12 @@ fi
 if [[ ! -z "${2:-}" ]]; then
   export MULTIZONE="${2}"
 fi
-export KUBE_DELETE_NODES=false
+if [[ ! -z "${3:-}" ]]; then
+  export KUBE_DELETE_NODES="${3}"
+fi
+if [[ ! -z "${4:-}" ]]; then
+  export KUBE_USE_EXISTING_MASTER="${4}"
+fi
 
 source "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"
 

test/e2e/ha_master.go

@@ -32,7 +32,7 @@ import (
 
 func addMasterReplica(zone string) error {
 	framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone))
-	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone)
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-grow-cluster.sh"), zone, "true", "true", "false")
 	framework.Logf("%s", v)
 	if err != nil {
 		return err
@@ -42,7 +42,27 @@ func addMasterReplica(zone string) error {
 
 func removeMasterReplica(zone string) error {
 	framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone))
-	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone, "true")
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-shrink-cluster.sh"), zone, "true", "false", "false")
+	framework.Logf("%s", v)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func addWorkerNodes(zone string) error {
+	framework.Logf(fmt.Sprintf("Adding worker nodes, zone: %s", zone))
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-grow-cluster.sh"), zone, "true", "false", "true")
+	framework.Logf("%s", v)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func removeWorkerNodes(zone string) error {
+	framework.Logf(fmt.Sprintf("Removing worker nodes, zone: %s", zone))
+	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-shrink-cluster.sh"), zone, "true", "true", "true")
 	framework.Logf("%s", v)
 	if err != nil {
 		return err
@@ -115,6 +135,7 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 	var c clientset.Interface
 	var ns string
 	var additionalReplicaZones []string
+	var additionalNodesZones []string
 	var existingRCs []string
 
 	BeforeEach(func() {
@@ -127,6 +148,11 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 	})
 
 	AfterEach(func() {
+		// Clean-up additional worker nodes if the test execution was broken.
+		for _, zone := range additionalNodesZones {
+			removeWorkerNodes(zone)
+		}
+
 		// Clean-up additional master replicas if the test execution was broken.
 		for _, zone := range additionalReplicaZones {
 			removeMasterReplica(zone)
@@ -140,6 +166,8 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		None Action = iota
 		AddReplica
 		RemoveReplica
+		AddNodes
+		RemoveNodes
 	)
 
 	step := func(action Action, zone string) {
@@ -151,9 +179,16 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 		case RemoveReplica:
 			framework.ExpectNoError(removeMasterReplica(zone))
 			additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone)
+		case AddNodes:
+			framework.ExpectNoError(addWorkerNodes(zone))
+			additionalNodesZones = append(additionalNodesZones, zone)
+		case RemoveNodes:
+			framework.ExpectNoError(removeWorkerNodes(zone))
+			additionalNodesZones = removeZoneFromZones(additionalNodesZones, zone)
 		}
 		verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1)
 		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, len(additionalReplicaZones)+1, 10*time.Minute)
+		framework.AllNodesReady(c, 5*time.Minute)
 
 		// Verify that API server works correctly with HA master.
 		rcName := "ha-master-" + strconv.Itoa(len(existingRCs))
@@ -191,4 +226,37 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
 			step(RemoveReplica, zones[i%len(zones)])
 		}
 	})
+
+	It("survive addition/removal replicas multizone workers [Serial][Disruptive]", func() {
+		zone := framework.TestContext.CloudConfig.Zone
+		region := findRegionForZone(zone)
+		zones := findZonesForRegion(region)
+		zones = removeZoneFromZones(zones, zone)
+
+		step(None, "")
+		numAdditionalReplicas := 2
+
+		// Add worker nodes.
+		for i := 0; i < numAdditionalReplicas && i < len(zones); i++ {
+			step(AddNodes, zones[i])
+		}
+
+		// Add master repilcas.
+		//
+		// If numAdditionalReplicas is larger then the number of remaining zones in the region,
+		// we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones.
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(AddReplica, zones[i%len(zones)])
+		}
+
+		// Remove master repilcas.
+		for i := 0; i < numAdditionalReplicas; i++ {
+			step(RemoveReplica, zones[i%len(zones)])
+		}
+
+		// Remove worker nodes.
+		for i := 0; i < numAdditionalReplicas && i < len(zones); i++ {
+			step(RemoveNodes, zones[i])
+		}
+	})
 })