Add integration test for deployment

hack/.linted_packages

@@ -438,6 +438,7 @@ test/integration/apiserver
 test/integration/client
 test/integration/configmap
 test/integration/defaulttolerationseconds
+test/integration/deployment
 test/integration/etcd
 test/integration/examples
 test/integration/federation

test/integration/BUILD

@@ -39,6 +39,7 @@ filegroup(
         "//test/integration/client:all-srcs",
         "//test/integration/configmap:all-srcs",
         "//test/integration/defaulttolerationseconds:all-srcs",
+        "//test/integration/deployment:all-srcs",
         "//test/integration/etcd:all-srcs",
         "//test/integration/evictions:all-srcs",
         "//test/integration/examples:all-srcs",

test/integration/deployment/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["deployment_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/v1:go_default_library",
+        "//pkg/controller/deployment/util:go_default_library",
+        "//test/integration/framework:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["util.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/v1:go_default_library",
+        "//pkg/api/v1/pod:go_default_library",
+        "//pkg/apis/extensions/v1beta1:go_default_library",
+        "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/informers/informers_generated/externalversions:go_default_library",
+        "//pkg/controller/deployment:go_default_library",
+        "//pkg/controller/deployment/util:go_default_library",
+        "//pkg/controller/replicaset:go_default_library",
+        "//pkg/util/labels:go_default_library",
+        "//test/integration/framework:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/integration/deployment/OWNERS

@@ -0,0 +1,6 @@
+reviewers:
+  - janetkuo
+  - kargakis
+approvers:
+  - janetkuo
+  - kargakis

test/integration/deployment/deployment_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployment
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+func TestNewDeployment(t *testing.T) {
+	s, closeFn, rm, dc, informers, c := dcSetup(t)
+	defer closeFn()
+	name := "test-new-deployment"
+	ns := framework.CreateTestingNamespace(name, s, t)
+	defer framework.DeleteTestingNamespace(ns, s, t)
+
+	replicas := int32(20)
+	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
+	tester.deployment.Spec.MinReadySeconds = 4
+
+	tester.deployment.Annotations = map[string]string{"test": "should-copy-to-replica-set", v1.LastAppliedConfigAnnotation: "should-not-copy-to-replica-set"}
+	deploy, err := c.Extensions().Deployments(ns.Name).Create(tester.deployment)
+	if err != nil {
+		t.Fatalf("failed to create deployment %s: %v", deploy.Name, err)
+	}
+
+	// Start informer and controllers
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+	informers.Start(stopCh)
+	go rm.Run(5, stopCh)
+	go dc.Run(5, stopCh)
+
+	// Wait for the Deployment to be updated to revision 1
+	err = tester.waitForDeploymentRevisionAndImage("1", fakeImage)
+	if err != nil {
+		t.Fatalf("failed to wait for Deployment revision %s: %v", deploy.Name, err)
+	}
+
+	// Make sure the Deployment status becomes valid while manually marking Deployment pods as ready at the same time
+	tester.waitForDeploymentStatusValidAndMarkPodsReady()
+
+	// Check new RS annotations
+	newRS, err := deploymentutil.GetNewReplicaSet(deploy, c)
+	if err != nil {
+		t.Fatalf("failed to get new ReplicaSet of Deployment %s: %v", deploy.Name, err)
+	}
+	if newRS.Annotations["test"] != "should-copy-to-replica-set" {
+		t.Errorf("expected new ReplicaSet annotations copied from Deployment %s, got: %v", deploy.Name, newRS.Annotations)
+	}
+	if newRS.Annotations[v1.LastAppliedConfigAnnotation] != "" {
+		t.Errorf("expected new ReplicaSet last-applied annotation not copied from Deployment %s", deploy.Name)
+	}
+}

test/integration/deployment/util.go

@@ -0,0 +1,353 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployment
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/api/v1"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+	"k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
+	"k8s.io/kubernetes/pkg/controller/deployment"
+	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
+	"k8s.io/kubernetes/pkg/controller/replicaset"
+	labelsutil "k8s.io/kubernetes/pkg/util/labels"
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+const (
+	pollInterval = 1 * time.Second
+	pollTimeout  = 60 * time.Second
+
+	fakeImageName = "fake-name"
+	fakeImage     = "fakeimage"
+)
+
+type deploymentTester struct {
+	t          *testing.T
+	c          clientset.Interface
+	deployment *v1beta1.Deployment
+}
+
+func testLabels() map[string]string {
+	return map[string]string{"name": "test"}
+}
+
+// newDeployment returns a RollingUpdate Deployment with with a fake container image
+func newDeployment(name, ns string, replicas int32) *v1beta1.Deployment {
+	return &v1beta1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "extensions/v1beta1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{MatchLabels: testLabels()},
+			Strategy: v1beta1.DeploymentStrategy{
+				Type: v1beta1.RollingUpdateDeploymentStrategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: testLabels(),
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  fakeImageName,
+							Image: fakeImage,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// dcSetup sets up necessities for Deployment integration test, including master, apiserver, informers, and clientset
+func dcSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *replicaset.ReplicaSetController, *deployment.DeploymentController, informers.SharedInformerFactory, clientset.Interface) {
+	masterConfig := framework.NewIntegrationTestMasterConfig()
+	_, s, closeFn := framework.RunAMaster(masterConfig)
+
+	config := restclient.Config{Host: s.URL}
+	clientSet, err := clientset.NewForConfig(&config)
+	if err != nil {
+		t.Fatalf("error in create clientset: %v", err)
+	}
+	resyncPeriod := 12 * time.Hour
+	informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "deployment-informers")), resyncPeriod)
+
+	dc := deployment.NewDeploymentController(
+		informers.Extensions().V1beta1().Deployments(),
+		informers.Extensions().V1beta1().ReplicaSets(),
+		informers.Core().V1().Pods(),
+		clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "deployment-controller")),
+	)
+	rm := replicaset.NewReplicaSetController(
+		informers.Extensions().V1beta1().ReplicaSets(),
+		informers.Core().V1().Pods(),
+		clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "replicaset-controller")),
+		replicaset.BurstReplicas,
+	)
+	return s, closeFn, rm, dc, informers, clientSet
+}
+
+// addPodConditionReady sets given pod status to ready at given time
+func addPodConditionReady(pod *v1.Pod, time metav1.Time) {
+	pod.Status = v1.PodStatus{
+		Phase: v1.PodRunning,
+		Conditions: []v1.PodCondition{
+			{
+				Type:               v1.PodReady,
+				Status:             v1.ConditionTrue,
+				LastTransitionTime: time,
+			},
+		},
+	}
+}
+
+func (d *deploymentTester) logReplicaSetsOfDeployment(allOldRSs []*v1beta1.ReplicaSet, newRS *v1beta1.ReplicaSet) {
+	if newRS != nil {
+		d.t.Logf("New ReplicaSet of Deployment %s:\n%+v", d.deployment.Name, *newRS)
+	} else {
+		d.t.Logf("New ReplicaSet of Deployment %s is nil.", d.deployment.Name)
+	}
+	if len(allOldRSs) > 0 {
+		d.t.Logf("All old ReplicaSets of Deployment %s:", d.deployment.Name)
+	}
+	for i := range allOldRSs {
+		d.t.Logf(spew.Sprintf("%#v", *allOldRSs[i]))
+	}
+}
+
+func (d *deploymentTester) logPodsOfDeployment(rsList []*v1beta1.ReplicaSet) {
+	minReadySeconds := d.deployment.Spec.MinReadySeconds
+	podListFunc := func(namespace string, options metav1.ListOptions) (*v1.PodList, error) {
+		return d.c.Core().Pods(namespace).List(options)
+	}
+
+	podList, err := deploymentutil.ListPods(d.deployment, rsList, podListFunc)
+
+	if err != nil {
+		d.t.Logf("Failed to list Pods of Deployment %s: %v", d.deployment.Name, err)
+		return
+	}
+	for _, pod := range podList.Items {
+		availability := "not available"
+		if podutil.IsPodAvailable(&pod, minReadySeconds, metav1.Now()) {
+			availability = "available"
+		}
+		d.t.Logf("Pod %s is %s:\n%s", pod.Name, availability, spew.Sprintf("%#v", pod))
+	}
+}
+
+// WaitForDeploymentRevisionAndImage waits for the deployment's and its new RS's revision and container image to match the given revision and image.
+// Note that deployment revision and its new RS revision should be updated shortly, so we only wait for 1 minute here to fail early.
+func (d *deploymentTester) waitForDeploymentRevisionAndImage(revision, image string) error {
+	var deployment *v1beta1.Deployment
+	var newRS *v1beta1.ReplicaSet
+	var reason string
+	deploymentName, ns := d.deployment.Name, d.deployment.Namespace
+	err := wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
+		var err error
+		deployment, err = d.c.Extensions().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		// The new ReplicaSet needs to be non-nil and contain the pod-template-hash label
+		newRS, err = deploymentutil.GetNewReplicaSet(deployment, d.c)
+		if err != nil {
+			return false, err
+		}
+		if newRS == nil {
+			reason = fmt.Sprintf("New replica set for deployment %q is yet to be created", deployment.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		if !labelsutil.SelectorHasLabel(newRS.Spec.Selector, v1beta1.DefaultDeploymentUniqueLabelKey) {
+			reason = fmt.Sprintf("New replica set %q doesn't have DefaultDeploymentUniqueLabelKey", newRS.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		// Check revision of this deployment, and of the new replica set of this deployment
+		if deployment.Annotations == nil || deployment.Annotations[deploymentutil.RevisionAnnotation] != revision {
+			reason = fmt.Sprintf("Deployment %q doesn't have the required revision set", deployment.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		if deployment.Spec.Template.Spec.Containers[0].Image != image {
+			reason = fmt.Sprintf("Deployment %q doesn't have the required image set", deployment.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		if newRS.Annotations == nil || newRS.Annotations[deploymentutil.RevisionAnnotation] != revision {
+			reason = fmt.Sprintf("New replica set %q doesn't have the required revision set", newRS.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		if newRS.Spec.Template.Spec.Containers[0].Image != image {
+			reason = fmt.Sprintf("New replica set %q doesn't have the required image set", newRS.Name)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		d.logReplicaSetsOfDeployment(nil, newRS)
+		err = fmt.Errorf(reason)
+	}
+	if newRS == nil {
+		return fmt.Errorf("deployment %q failed to create new replica set", deploymentName)
+	}
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %q (got %s / %s) and new replica set %q (got %s / %s) revision and image to match expectation (expected %s / %s): %v", deploymentName, deployment.Annotations[deploymentutil.RevisionAnnotation], deployment.Spec.Template.Spec.Containers[0].Image, newRS.Name, newRS.Annotations[deploymentutil.RevisionAnnotation], newRS.Spec.Template.Spec.Containers[0].Image, revision, image, err)
+	}
+	return nil
+}
+
+// markAllPodsReady manually updates all Deployment pods status to ready
+func (d *deploymentTester) markAllPodsReady() {
+	ns := d.deployment.Namespace
+	selector, err := metav1.LabelSelectorAsSelector(d.deployment.Spec.Selector)
+	if err != nil {
+		d.t.Fatalf("failed to parse Deployment selector: %v", err)
+	}
+	var readyPods int32
+	err = wait.Poll(100*time.Millisecond, pollTimeout, func() (bool, error) {
+		readyPods = 0
+		pods, err := d.c.Core().Pods(ns).List(metav1.ListOptions{LabelSelector: selector.String()})
+		if err != nil {
+			d.t.Logf("failed to list Deployment pods, will retry later: %v", err)
+			return false, nil
+		}
+		for i := range pods.Items {
+			pod := pods.Items[i]
+			if podutil.IsPodReady(&pod) {
+				readyPods++
+				continue
+			}
+			addPodConditionReady(&pod, metav1.Now())
+			if _, err = d.c.Core().Pods(ns).UpdateStatus(&pod); err != nil {
+				d.t.Logf("failed to update Deployment pod %s, will retry later: %v", pod.Name, err)
+			} else {
+				readyPods++
+			}
+		}
+		if readyPods >= *d.deployment.Spec.Replicas {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		d.t.Fatalf("failed to mark all Deployment pods to ready: %v", err)
+	}
+}
+
+func (d *deploymentTester) waitForDeploymentStatusValid() error {
+	var (
+		oldRSs, allOldRSs, allRSs []*v1beta1.ReplicaSet
+		newRS                     *v1beta1.ReplicaSet
+		deployment                *v1beta1.Deployment
+		reason                    string
+	)
+
+	name := d.deployment.Name
+	err := wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
+		var err error
+		deployment, err = d.c.Extensions().Deployments(d.deployment.Namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		oldRSs, allOldRSs, newRS, err = deploymentutil.GetAllReplicaSets(deployment, d.c)
+		if err != nil {
+			return false, err
+		}
+		if newRS == nil {
+			// New RC hasn't been created yet.
+			reason = "new replica set hasn't been created yet"
+			d.t.Logf(reason)
+			return false, nil
+		}
+		allRSs = append(oldRSs, newRS)
+		// The old/new ReplicaSets need to contain the pod-template-hash label
+		for i := range allRSs {
+			if !labelsutil.SelectorHasLabel(allRSs[i].Spec.Selector, v1beta1.DefaultDeploymentUniqueLabelKey) {
+				reason = "all replica sets need to contain the pod-template-hash label"
+				d.t.Logf(reason)
+				return false, nil
+			}
+		}
+		totalCreated := deploymentutil.GetReplicaCountForReplicaSets(allRSs)
+		maxCreated := *(deployment.Spec.Replicas) + deploymentutil.MaxSurge(*deployment)
+		if totalCreated > maxCreated {
+			reason = fmt.Sprintf("total pods created: %d, more than the max allowed: %d", totalCreated, maxCreated)
+			d.t.Logf(reason)
+			return false, nil
+		}
+		minAvailable := deploymentutil.MinAvailable(deployment)
+		if deployment.Status.AvailableReplicas < minAvailable {
+			reason = fmt.Sprintf("total pods available: %d, less than the min required: %d", deployment.Status.AvailableReplicas, minAvailable)
+			d.t.Logf(reason)
+			return false, nil
+		}
+
+		// When the deployment status and its underlying resources reach the desired state, we're done
+		if deploymentutil.DeploymentComplete(deployment, &deployment.Status) {
+			return true, nil
+		}
+
+		reason = fmt.Sprintf("deployment status: %#v", deployment.Status)
+		d.t.Logf(reason)
+
+		return false, nil
+	})
+
+	if err == wait.ErrWaitTimeout {
+		d.logReplicaSetsOfDeployment(allOldRSs, newRS)
+		d.logPodsOfDeployment(allRSs)
+		err = fmt.Errorf("%s", reason)
+	}
+	if err != nil {
+		return fmt.Errorf("error waiting for deployment %q status to match expectation: %v", d.deployment.Name, err)
+	}
+	return nil
+}
+
+// waitForDeploymentStatusValidAndMarkPodsReady waits for the Deployment status to become valid
+// while marking all Deployment pods as ready at the same time.
+func (d *deploymentTester) waitForDeploymentStatusValidAndMarkPodsReady() {
+	// Manually mark all Deployment pods as ready in a separate goroutine
+	go d.markAllPodsReady()
+
+	// Make sure the Deployment status is valid while Deployment pods are becoming ready
+	err := d.waitForDeploymentStatusValid()
+	if err != nil {
+		d.t.Fatalf("failed to wait for Deployment status %s: %v", d.deployment.Name, err)
+	}
+}