Merge pull request #21030 from janetkuo/deployment-label-adopted

Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot 2016-02-22 23:45:42 -08:00
commit 0afbc71f31
5 changed files with 365 additions and 52 deletions

View File

@ -665,6 +665,10 @@ func (dc *DeploymentController) getNewReplicaSet(deployment extensions.Deploymen
newRevision := strconv.FormatInt(maxOldRevision+1, 10)
existingNewRS, err := deploymentutil.GetNewReplicaSetFromList(deployment, dc.client,
func(namespace string, options api.ListOptions) (*api.PodList, error) {
podList, err := dc.podStore.Pods(namespace).List(options.LabelSelector)
return &podList, err
},
func(namespace string, options api.ListOptions) ([]extensions.ReplicaSet, error) {
return dc.rsStore.ReplicaSets(namespace).List(options.LabelSelector)
})

View File

@ -25,11 +25,14 @@ import (
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
unversionedcore "k8s.io/kubernetes/pkg/client/typed/generated/core/unversioned"
unversionedextensions "k8s.io/kubernetes/pkg/client/typed/generated/extensions/unversioned"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/integer"
intstrutil "k8s.io/kubernetes/pkg/util/intstr"
labelsutil "k8s.io/kubernetes/pkg/util/labels"
podutil "k8s.io/kubernetes/pkg/util/pod"
"k8s.io/kubernetes/pkg/util/wait"
)
const (
@ -55,28 +58,21 @@ func GetOldReplicaSets(deployment extensions.Deployment, c clientset.Interface)
})
}
// TODO: switch this to full namespacers
type rsListFunc func(string, api.ListOptions) ([]extensions.ReplicaSet, error)
type podListFunc func(string, api.ListOptions) (*api.PodList, error)
// GetOldReplicaSetsFromLists returns two sets of old replica sets targeted by the given Deployment; get PodList and ReplicaSetList with input functions.
// Note that the first set of old replica sets doesn't include the ones with no pods, and the second set of old replica sets include all old replica sets.
func GetOldReplicaSetsFromLists(deployment extensions.Deployment, c clientset.Interface, getPodList func(string, api.ListOptions) (*api.PodList, error), getRSList func(string, api.ListOptions) ([]extensions.ReplicaSet, error)) ([]*extensions.ReplicaSet, []*extensions.ReplicaSet, error) {
namespace := deployment.ObjectMeta.Namespace
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
if err != nil {
return nil, nil, fmt.Errorf("invalid label selector: %v", err)
}
// 1. Find all pods whose labels match deployment.Spec.Selector
options := api.ListOptions{LabelSelector: selector}
podList, err := getPodList(namespace, options)
if err != nil {
return nil, nil, fmt.Errorf("error listing pods: %v", err)
}
// 2. Find the corresponding replica sets for pods in podList.
func GetOldReplicaSetsFromLists(deployment extensions.Deployment, c clientset.Interface, getPodList podListFunc, getRSList rsListFunc) ([]*extensions.ReplicaSet, []*extensions.ReplicaSet, error) {
// Find all pods whose labels match deployment.Spec.Selector, and corresponding replica sets for pods in podList.
// All pods and replica sets are labeled with pod-template-hash to prevent overlapping
// TODO: Right now we list all replica sets and then filter. We should add an API for this.
oldRSs := map[string]extensions.ReplicaSet{}
allOldRSs := map[string]extensions.ReplicaSet{}
rsList, err := getRSList(namespace, options)
rsList, podList, err := rsAndPodsWithHashKeySynced(deployment, c, getRSList, getPodList)
if err != nil {
return nil, nil, fmt.Errorf("error listing replica sets: %v", err)
return nil, nil, fmt.Errorf("error labeling replica sets and pods with pod-template-hash: %v", err)
}
newRSTemplate := GetNewReplicaSetTemplate(deployment)
for _, pod := range podList.Items {
@ -113,6 +109,9 @@ func GetOldReplicaSetsFromLists(deployment extensions.Deployment, c clientset.In
// Returns nil if the new replica set doesn't exist yet.
func GetNewReplicaSet(deployment extensions.Deployment, c clientset.Interface) (*extensions.ReplicaSet, error) {
return GetNewReplicaSetFromList(deployment, c,
func(namespace string, options api.ListOptions) (*api.PodList, error) {
return c.Core().Pods(namespace).List(options)
},
func(namespace string, options api.ListOptions) ([]extensions.ReplicaSet, error) {
rsList, err := c.Extensions().ReplicaSets(namespace).List(options)
return rsList.Items, err
@ -121,14 +120,8 @@ func GetNewReplicaSet(deployment extensions.Deployment, c clientset.Interface) (
// GetNewReplicaSetFromList returns a replica set that matches the intent of the given deployment; get ReplicaSetList with the input function.
// Returns nil if the new replica set doesn't exist yet.
func GetNewReplicaSetFromList(deployment extensions.Deployment, c clientset.Interface, getRSList func(string, api.ListOptions) ([]extensions.ReplicaSet, error)) (*extensions.ReplicaSet, error) {
namespace := deployment.ObjectMeta.Namespace
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("invalid label selector: %v", err)
}
rsList, err := getRSList(namespace, api.ListOptions{LabelSelector: selector})
func GetNewReplicaSetFromList(deployment extensions.Deployment, c clientset.Interface, getPodList podListFunc, getRSList rsListFunc) (*extensions.ReplicaSet, error) {
rsList, _, err := rsAndPodsWithHashKeySynced(deployment, c, getRSList, getPodList)
if err != nil {
return nil, fmt.Errorf("error listing ReplicaSets: %v", err)
}
@ -144,6 +137,166 @@ func GetNewReplicaSetFromList(deployment extensions.Deployment, c clientset.Inte
return nil, nil
}
// rsAndPodsWithHashKeySynced returns the RSs and pods the given deployment targets, with pod-template-hash information synced.
func rsAndPodsWithHashKeySynced(deployment extensions.Deployment, c clientset.Interface, getRSList rsListFunc, getPodList podListFunc) ([]extensions.ReplicaSet, *api.PodList, error) {
namespace := deployment.Namespace
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
if err != nil {
return nil, nil, err
}
options := api.ListOptions{LabelSelector: selector}
rsList, err := getRSList(namespace, options)
if err != nil {
return nil, nil, err
}
syncedRSList := []extensions.ReplicaSet{}
for _, rs := range rsList {
// Add pod-template-hash information if it's not in the RS.
// Otherwise, new RS produced by Deployment will overlap with pre-existing ones
// that aren't constrained by the pod-template-hash.
syncedRS, err := addHashKeyToRSAndPods(deployment, c, rs, getPodList)
if err != nil {
return nil, nil, err
}
syncedRSList = append(syncedRSList, *syncedRS)
}
syncedPodList, err := getPodList(namespace, options)
if err != nil {
return nil, nil, err
}
return syncedRSList, syncedPodList, nil
}
// addHashKeyToRSAndPods adds pod-template-hash information to the given rs, if it's not already there, with the following steps:
// 1. Add hash label to the rs's pod template, and make sure the controller sees this update so that no orphaned pods will be created
// 2. Add hash label to all pods this rs owns
// 3. Add hash label to the rs's label and selector
func addHashKeyToRSAndPods(deployment extensions.Deployment, c clientset.Interface, rs extensions.ReplicaSet, getPodList podListFunc) (updatedRS *extensions.ReplicaSet, err error) {
// If the rs already has the new hash label in its selector, it's done syncing
namespace := deployment.Namespace
hash := fmt.Sprintf("%d", podutil.GetPodTemplateSpecHash(api.PodTemplateSpec{
ObjectMeta: rs.Spec.Template.ObjectMeta,
Spec: rs.Spec.Template.Spec,
}))
if labelsutil.SelectorHasLabel(rs.Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) {
return &rs, nil
}
// 1. Add hash template label to the rs. This ensures that any newly created pods will have the new label.
if len(rs.Spec.Template.Labels[extensions.DefaultDeploymentUniqueLabelKey]) == 0 {
updatedRS, err = updateRSWithRetries(c.Extensions().ReplicaSets(namespace), &rs, func(updated *extensions.ReplicaSet) {
updated.Spec.Template.Labels = labelsutil.AddLabel(updated.Spec.Template.Labels, extensions.DefaultDeploymentUniqueLabelKey, hash)
})
if err != nil {
return nil, err
}
}
// Make sure rs pod template is updated so that it won't create pods without the new label (orphaned pods).
if updatedRS.Generation > updatedRS.Status.ObservedGeneration {
if err = waitForReplicaSetUpdated(c, updatedRS.Generation, namespace, rs.Name); err != nil {
return nil, err
}
}
// 2. Update all pods managed by the rs to have the new hash label, so they will be correctly adopted.
selector, err := unversioned.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
return nil, err
}
options := api.ListOptions{LabelSelector: selector}
podList, err := getPodList(namespace, options)
if err != nil {
return nil, err
}
if err = labelPodsWithHash(podList, c, namespace, hash); err != nil {
return nil, err
}
// 3. Update rs label and selector to include the new hash label
// Copy the old selector, so that we can scrub out any orphaned pods
if updatedRS, err = updateRSWithRetries(c.Extensions().ReplicaSets(namespace), &rs, func(updated *extensions.ReplicaSet) {
updated.Labels = labelsutil.AddLabel(updated.Labels, extensions.DefaultDeploymentUniqueLabelKey, hash)
updated.Spec.Selector = labelsutil.AddLabelToSelector(updated.Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey, hash)
}); err != nil {
return nil, err
}
// TODO: look for orphaned pods and label them in the background somewhere else periodically
return updatedRS, nil
}
func waitForReplicaSetUpdated(c clientset.Interface, desiredGeneration int64, namespace, name string) error {
return wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
rs, err := c.Extensions().ReplicaSets(namespace).Get(name)
if err != nil {
return false, err
}
return rs.Status.ObservedGeneration >= desiredGeneration, nil
})
}
// labelPodsWithHash labels all pods in the given podList with the new hash label.
func labelPodsWithHash(podList *api.PodList, c clientset.Interface, namespace, hash string) error {
for _, pod := range podList.Items {
// Only label the pod that doesn't already have the new hash
if pod.Labels[extensions.DefaultDeploymentUniqueLabelKey] != hash {
if _, err := updatePodWithRetries(c.Core().Pods(namespace), &pod, func(updated *api.Pod) {
pod.Labels = labelsutil.AddLabel(pod.Labels, extensions.DefaultDeploymentUniqueLabelKey, hash)
}); err != nil {
return err
}
}
}
return nil
}
// TODO: use client library instead when it starts to support update retries
// see https://github.com/kubernetes/kubernetes/issues/21479
type updateRSFunc func(rs *extensions.ReplicaSet)
func updateRSWithRetries(rsClient unversionedextensions.ReplicaSetInterface, rs *extensions.ReplicaSet, applyUpdate updateRSFunc) (*extensions.ReplicaSet, error) {
var err error
oldRs := rs
err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
// Apply the update, then attempt to push it to the apiserver.
applyUpdate(rs)
if rs, err = rsClient.Update(rs); err == nil {
// rs contains the latest controller post update
return true, nil
}
// Update the controller with the latest resource version, if the update failed we
// can't trust rs so use oldRs.Name.
if rs, err = rsClient.Get(oldRs.Name); err != nil {
// The Get failed: Value in rs cannot be trusted.
rs = oldRs
}
// The Get passed: rs contains the latest controller, expect a poll for the update.
return false, nil
})
// If the error is non-nil the returned controller cannot be trusted, if it is nil, the returned
// controller contains the applied update.
return rs, err
}
type updatePodFunc func(pod *api.Pod)
func updatePodWithRetries(podClient unversionedcore.PodInterface, pod *api.Pod, applyUpdate updatePodFunc) (*api.Pod, error) {
var err error
oldPod := pod
err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
// Apply the update, then attempt to push it to the apiserver.
applyUpdate(pod)
if pod, err = podClient.Update(pod); err == nil {
return true, nil
}
if pod, err = podClient.Get(oldPod.Name); err != nil {
pod = oldPod
}
return false, nil
})
return pod, err
}
// Returns the desired PodTemplateSpec for the new ReplicaSet corresponding to the given ReplicaSet.
func GetNewReplicaSetTemplate(deployment extensions.Deployment) api.PodTemplateSpec {
// newRS will have the same template as in deployment spec, plus a unique label in some cases.

View File

@ -22,14 +22,44 @@ import (
"time"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
"k8s.io/kubernetes/pkg/client/unversioned/testclient/simple"
"k8s.io/kubernetes/pkg/client/testing/core"
"k8s.io/kubernetes/pkg/client/unversioned/testclient"
"k8s.io/kubernetes/pkg/runtime"
)
func addListRSReactor(fakeClient *fake.Clientset, obj runtime.Object) *fake.Clientset {
fakeClient.AddReactor("list", "replicasets", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, obj, nil
})
return fakeClient
}
func addListPodsReactor(fakeClient *fake.Clientset, obj runtime.Object) *fake.Clientset {
fakeClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, obj, nil
})
return fakeClient
}
func addUpdateRSReactor(fakeClient *fake.Clientset) *fake.Clientset {
fakeClient.AddReactor("update", "replicasets", func(action core.Action) (handled bool, ret runtime.Object, err error) {
obj := action.(testclient.UpdateAction).GetObject().(*extensions.ReplicaSet)
return true, obj, nil
})
return fakeClient
}
func addUpdatePodsReactor(fakeClient *fake.Clientset) *fake.Clientset {
fakeClient.AddReactor("update", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
obj := action.(testclient.UpdateAction).GetObject().(*api.Pod)
return true, obj, nil
})
return fakeClient
}
func newPod(now time.Time, ready bool, beforeSec int) api.Pod {
conditionStatus := api.ConditionFalse
if ready {
@ -190,47 +220,47 @@ func TestGetNewRC(t *testing.T) {
tests := []struct {
test string
rsList extensions.ReplicaSetList
objs []runtime.Object
expected *extensions.ReplicaSet
}{
{
"No new ReplicaSet",
extensions.ReplicaSetList{
Items: []extensions.ReplicaSet{
generateRS(generateDeployment("foo")),
generateRS(generateDeployment("bar")),
[]runtime.Object{
&api.PodList{},
&extensions.ReplicaSetList{
Items: []extensions.ReplicaSet{
generateRS(generateDeployment("foo")),
generateRS(generateDeployment("bar")),
},
},
},
nil,
},
{
"Has new ReplicaSet",
extensions.ReplicaSetList{
Items: []extensions.ReplicaSet{
generateRS(generateDeployment("foo")),
generateRS(generateDeployment("bar")),
generateRS(generateDeployment("abc")),
newRC,
generateRS(generateDeployment("xyz")),
[]runtime.Object{
&api.PodList{},
&extensions.ReplicaSetList{
Items: []extensions.ReplicaSet{
generateRS(generateDeployment("foo")),
generateRS(generateDeployment("bar")),
generateRS(generateDeployment("abc")),
newRC,
generateRS(generateDeployment("xyz")),
},
},
},
&newRC,
},
}
ns := api.NamespaceDefault
for _, test := range tests {
c := &simple.Client{
Request: simple.Request{
Method: "GET",
Path: testapi.Default.ResourcePath("replicaSets", ns, ""),
},
Response: simple.Response{
StatusCode: 200,
Body: &test.rsList,
},
}
rs, err := GetNewReplicaSet(newDeployment, c.Setup(t).Clientset)
fakeClient := &fake.Clientset{}
fakeClient = addListPodsReactor(fakeClient, test.objs[0])
fakeClient = addListRSReactor(fakeClient, test.objs[1])
fakeClient = addUpdatePodsReactor(fakeClient)
fakeClient = addUpdateRSReactor(fakeClient)
rs, err := GetNewReplicaSet(newDeployment, fakeClient)
if err != nil {
t.Errorf("In test case %s, got unexpected error %v", test.test, err)
}
@ -313,12 +343,24 @@ func TestGetOldRCs(t *testing.T) {
}
for _, test := range tests {
rss, _, err := GetOldReplicaSets(newDeployment, fake.NewSimpleClientset(test.objs...))
fakeClient := &fake.Clientset{}
fakeClient = addListPodsReactor(fakeClient, test.objs[0])
fakeClient = addListRSReactor(fakeClient, test.objs[1])
fakeClient = addUpdatePodsReactor(fakeClient)
fakeClient = addUpdateRSReactor(fakeClient)
rss, _, err := GetOldReplicaSets(newDeployment, fakeClient)
if err != nil {
t.Errorf("In test case %s, got unexpected error %v", test.test, err)
}
if !equal(rss, test.expected) {
t.Errorf("In test case %q, expected %v, got %v", test.test, test.expected, rss)
t.Errorf("In test case %q, expected:", test.test)
for _, rs := range test.expected {
t.Errorf("rs = %+v", rs)
}
t.Errorf("In test case %q, got:", test.test)
for _, rs := range rss {
t.Errorf("rs = %+v", rs)
}
}
}
}

View File

@ -54,6 +54,19 @@ func CloneAndRemoveLabel(labels map[string]string, labelKey string) map[string]s
return newLabels
}
// AddLabel returns a map with the given key and value added to the given map.
func AddLabel(labels map[string]string, labelKey string, labelValue string) map[string]string {
if labelKey == "" {
// Dont need to add a label.
return labels
}
if labels == nil {
labels = make(map[string]string)
}
labels[labelKey] = labelValue
return labels
}
// Clones the given selector and returns a new selector with the given key and value added.
// Returns the given selector, if labelKey is empty.
func CloneSelectorAndAddLabel(selector *unversioned.LabelSelector, labelKey string, labelValue uint32) *unversioned.LabelSelector {
@ -93,3 +106,21 @@ func CloneSelectorAndAddLabel(selector *unversioned.LabelSelector, labelKey stri
return newSelector
}
// AddLabelToSelector returns a selector with the given key and value added to the given selector's MatchLabels.
func AddLabelToSelector(selector *unversioned.LabelSelector, labelKey string, labelValue string) *unversioned.LabelSelector {
if labelKey == "" {
// Dont need to add a label.
return selector
}
if selector.MatchLabels == nil {
selector.MatchLabels = make(map[string]string)
}
selector.MatchLabels[labelKey] = labelValue
return selector
}
// SelectorHasLabel checks if the given selector contains the given label key in its MatchLabels
func SelectorHasLabel(selector *unversioned.LabelSelector, labelKey string) bool {
return len(selector.MatchLabels[labelKey]) > 0
}

View File

@ -66,6 +66,9 @@ var _ = Describe("Deployment", func() {
It("[Flaky] deployment should support rollback when there's replica set with no revision", func() {
testRollbackDeploymentRSNoRevision(f)
})
It("deployment should label adopted RSs and pods", func() {
testDeploymentLabelAdopted(f)
})
})
func newRS(rsName string, replicas int, rsPodLabels map[string]string, imageName string, image string) *extensions.ReplicaSet {
@ -272,6 +275,17 @@ func testRollingUpdateDeployment(f *Framework) {
// Check if it's updated to revision 1 correctly
checkDeploymentRevision(c, ns, deploymentName, "1", "redis", "redis")
// There should be 1 old RS (nginx-controller, which is adopted)
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
_, allOldRSs, err := deploymentutil.GetOldReplicaSets(*deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(len(allOldRSs)).Should(Equal(1))
// The old RS should contain pod-template-hash in its selector, label, and template label
Expect(len(allOldRSs[0].Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(allOldRSs[0].Spec.Selector.MatchLabels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(allOldRSs[0].Spec.Template.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
}
func testRollingUpdateDeploymentEvents(f *Framework) {
@ -796,3 +810,72 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
// Check if it's still revision 3 and still has the old pod template
checkDeploymentRevision(c, ns, deploymentName, "3", deploymentImageName, deploymentImage)
}
func testDeploymentLabelAdopted(f *Framework) {
ns := f.Namespace.Name
// TODO: remove unversionedClient when the refactoring is done. Currently some
// functions like verifyPod still expects a unversioned#Client.
unversionedClient := f.Client
c := clientset.FromUnversionedClient(unversionedClient)
// Create nginx pods.
podName := "nginx"
podLabels := map[string]string{"name": podName}
rsName := "nginx-controller"
replicas := 3
_, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, podLabels, podName, podName))
Expect(err).NotTo(HaveOccurred())
// Verify that the required pods have come up.
err = verifyPods(unversionedClient, ns, podName, false, 3)
if err != nil {
Logf("error in waiting for pods to come up: %s", err)
Expect(err).NotTo(HaveOccurred())
}
// Create a nginx deployment to adopt the old rs.
deploymentName := "nginx-deployment"
Logf("Creating deployment %s", deploymentName)
_, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, podLabels, podName, podName, extensions.RollingUpdateDeploymentStrategyType, nil))
Expect(err).NotTo(HaveOccurred())
defer func() {
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
Logf("deleting deployment %s", deploymentName)
Expect(c.Extensions().Deployments(ns).Delete(deploymentName, nil)).NotTo(HaveOccurred())
// TODO: remove this once we can delete replica sets with deployment
newRS, err := deploymentutil.GetNewReplicaSet(*deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(c.Extensions().ReplicaSets(ns).Delete(newRS.Name, nil)).NotTo(HaveOccurred())
}()
// The RS and pods should be relabeled before the status is updated by syncRollingUpdateDeployment
err = waitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0)
Expect(err).NotTo(HaveOccurred())
// Check if it's updated to revision 1 correctly
checkDeploymentRevision(c, ns, deploymentName, "1", "nginx", "nginx")
// There should be no old RSs (overlapping RS)
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
oldRSs, allOldRSs, err := deploymentutil.GetOldReplicaSets(*deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(len(oldRSs)).Should(Equal(0))
Expect(len(allOldRSs)).Should(Equal(0))
// New RS should contain pod-template-hash in its selector, label, and template label
newRS, err := deploymentutil.GetNewReplicaSet(*deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(len(newRS.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(newRS.Spec.Selector.MatchLabels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(newRS.Spec.Template.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
// All pods targeted by the deployment should contain pod-template-hash in their labels, and there should be only 3 pods
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
Expect(err).NotTo(HaveOccurred())
options := api.ListOptions{LabelSelector: selector}
pods, err := c.Core().Pods(ns).List(options)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
Expect(len(pod.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
}
Expect(len(pods.Items)).Should(Equal(replicas))
}