petset E2Es

test/e2e/petset.go

@@ -18,7 +18,10 @@ package e2e
 
 import (
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	inf "gopkg.in/inf.v0"
@@ -32,39 +35,55 @@ import (
 	"k8s.io/kubernetes/pkg/apis/apps"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/controller/petset"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
+	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 const (
 	petsetPoll = 10 * time.Second
-	petsetTimeout = 5 * time.Minute
+	// Some pets install base packages via wget
+	petsetTimeout           = 10 * time.Minute
+	zookeeperManifestPath   = "test/e2e/testing-manifests/petset/zookeeper"
+	mysqlGaleraManifestPath = "test/e2e/testing-manifests/petset/mysql-galera"
+	redisManifestPath       = "test/e2e/testing-manifests/petset/redis"
 )
 
 var _ = framework.KubeDescribe("PetSet", func() {
 	f := framework.NewDefaultFramework("petset")
+	var ns string
+	var c *client.Client
+
+	BeforeEach(func() {
+		// PetSet is in alpha, so it's disabled on all platforms except GCE.
+		// In theory, tests that restart pets should pass on any platform with a
+		// dynamic volume provisioner.
+		framework.SkipUnlessProviderIs("gce")
+
+		var err error
+		c, err = framework.LoadClient()
+		Expect(err).NotTo(HaveOccurred())
+		ns = f.Namespace.Name
+	})
+
+	framework.KubeDescribe("Basic PetSet functionality", func() {
 		psName := "pet"
 		labels := map[string]string{
 			"foo": "bar",
 			"baz": "blah",
 		}
 		headlessSvcName := "test"
-	var ns string
 
-	var c *client.Client
 		BeforeEach(func() {
-		var err error
-		c, err = framework.LoadClient()
-		Expect(err).NotTo(HaveOccurred())
-		ns = f.Namespace.Name
-
 			By("creating service " + headlessSvcName + " in namespace " + ns)
 			headlessService := createServiceSpec(headlessSvcName, true, labels)
-		_, err = c.Services(ns).Create(headlessService)
+			_, err := c.Services(ns).Create(headlessService)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-	It("provide basic identity [Feature:PetSet]", func() {
+		It("should provide basic identity [Feature:PetSet]", func() {
 			By("creating petset " + psName + " in namespace " + ns)
 			defer func() {
 				err := c.Apps().PetSets(ns).Delete(psName, nil)
@@ -77,22 +96,22 @@ var _ = framework.KubeDescribe("PetSet", func() {
 			_, err := c.Apps().PetSets(ns).Create(ps)
 			Expect(err).NotTo(HaveOccurred())
 
-		pt := petTester{c: c}
+			pst := petSetTester{c: c}
 
 			By("Saturating pet set " + ps.Name)
-		pt.saturate(ps)
+			pst.saturate(ps)
 
 			cmd := "echo $(hostname) > /data/hostname"
 			By("Running " + cmd + " in all pets")
-		pt.execInPets(ps, cmd)
+			pst.execInPets(ps, cmd)
 
 			By("Restarting pet set " + ps.Name)
-		pt.restart(ps)
+			pst.restart(ps)
-		pt.saturate(ps)
+			pst.saturate(ps)
 
 			cmd = "if [ \"$(cat /data/hostname)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi"
 			By("Running " + cmd + " in all pets")
-		pt.execInPets(ps, cmd)
+			pst.execInPets(ps, cmd)
 		})
 
 		It("should handle healthy pet restarts during scale [Feature:PetSet]", func() {
@@ -108,47 +127,291 @@ var _ = framework.KubeDescribe("PetSet", func() {
 			_, err := c.Apps().PetSets(ns).Create(ps)
 			Expect(err).NotTo(HaveOccurred())
 
-		pt := petTester{c: c}
+			pst := petSetTester{c: c}
-		pt.waitForRunning(1, ps)
+			pst.waitForRunning(1, ps)
 
 			By("Marking pet at index 0 as healthy.")
-		pt.setHealthy(ps)
+			pst.setHealthy(ps)
 
 			By("Waiting for pet at index 1 to enter running.")
-		pt.waitForRunning(2, ps)
+			pst.waitForRunning(2, ps)
 
 			// Now we have 1 healthy and 1 unhealthy pet. Deleting the healthy pet should *not*
 			// create a new pet till the remaining pet becomes healthy, which won't happen till
 			// we set the healthy bit.
 
 			By("Deleting healthy pet at index 0.")
-		pt.deletePetAtIndex(0, ps)
+			pst.deletePetAtIndex(0, ps)
 
 			By("Confirming pet at index 0 is not recreated.")
-		pt.confirmPetCount(1, ps, 10*time.Second)
+			pst.confirmPetCount(1, ps, 10*time.Second)
 
 			By("Deleting unhealthy pet at index 1.")
-		pt.deletePetAtIndex(1, ps)
+			pst.deletePetAtIndex(1, ps)
 
 			By("Confirming all pets in petset are created.")
-		pt.saturate(ps)
+			pst.saturate(ps)
 		})
 	})
 
-type petTester struct {
+	framework.KubeDescribe("Deploy clustered applications", func() {
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce")
+		})
+
+		AfterEach(func() {
+			// TODO: delete pvs
+			if !CurrentGinkgoTestDescription().Failed {
+				return
+			}
+			dumpDebugInfo(c, ns)
+		})
+
+		It("should creating a working zookeeper cluster [Feature:PetSet]", func() {
+			pst := &petSetTester{c: c}
+			pet := &zookeeperTester{tester: pst}
+			By("Deploying " + pet.name())
+			ps := pet.deploy(ns)
+
+			By("Creating foo:bar in member with index 0")
+			pet.write(0, map[string]string{"foo": "bar"})
+
+			By("Restarting pet set " + ps.Name)
+			pst.restart(ps)
+			pst.waitForRunning(ps.Spec.Replicas, ps)
+
+			By("Reading value under foo from member with index 2")
+			if v := pet.read(2, "foo"); v != "bar" {
+				framework.Failf("Read unexpected value %v, expected bar under key foo", v)
+			}
+		})
+
+		It("should creating a working redis cluster [Feature:PetSet]", func() {
+			pst := &petSetTester{c: c}
+			pet := &redisTester{tester: pst}
+			By("Deploying " + pet.name())
+			ps := pet.deploy(ns)
+
+			By("Creating foo:bar in member with index 0")
+			pet.write(0, map[string]string{"foo": "bar"})
+
+			By("Restarting pet set " + ps.Name)
+			pst.restart(ps)
+			pst.waitForRunning(ps.Spec.Replicas, ps)
+
+			By("Reading value under foo from member with index 2")
+			if v := pet.read(2, "foo"); v != "bar" {
+				framework.Failf("Read unexpected value %v, expected bar under key foo", v)
+			}
+		})
+
+		It("should creating a working mysql cluster [Feature:PetSet]", func() {
+			pst := &petSetTester{c: c}
+			pet := &mysqlGaleraTester{tester: pst}
+			By("Deploying " + pet.name())
+			ps := pet.deploy(ns)
+
+			By("Creating foo:bar in member with index 0")
+			pet.write(0, map[string]string{"foo": "bar"})
+
+			By("Restarting pet set " + ps.Name)
+			pst.restart(ps)
+			pst.waitForRunning(ps.Spec.Replicas, ps)
+
+			By("Reading value under foo from member with index 2")
+			if v := pet.read(2, "foo"); v != "bar" {
+				framework.Failf("Read unexpected value %v, expected bar under key foo", v)
+			}
+		})
+	})
+})
+
+func dumpDebugInfo(c *client.Client, ns string) {
+	pl, _ := c.Pods(ns).List(api.ListOptions{LabelSelector: labels.Everything()})
+	for _, p := range pl.Items {
+		desc, _ := framework.RunKubectl("describe", "po", p.Name, fmt.Sprintf("--namespace=%v", ns))
+		framework.Logf("\nOutput of kubectl describe %v:\n%v", p.Name, desc)
+
+		l, _ := framework.RunKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", ns), "--tail=100")
+		framework.Logf("\nLast 100 log lines of %v:\n%v", p.Name, l)
+	}
+}
+
+func kubectlExecWithRetries(args ...string) (out string) {
+	var err error
+	for i := 0; i < 3; i++ {
+		if out, err = framework.RunKubectl(args...); err == nil {
+			return
+		}
+		framework.Logf("Retrying %v:\nerror %v\nstdout %v", args, err, out)
+	}
+	framework.Failf("Failed to execute \"%v\" with retries: %v", args, err)
+	return
+}
+
+type petTester interface {
+	deploy(ns string) *apps.PetSet
+	write(petIndex int, kv map[string]string)
+	read(petIndex int, key string) string
+	name() string
+}
+
+type zookeeperTester struct {
+	ps     *apps.PetSet
+	tester *petSetTester
+}
+
+func (z *zookeeperTester) name() string {
+	return "zookeeper"
+}
+
+func (z *zookeeperTester) deploy(ns string) *apps.PetSet {
+	z.ps = z.tester.createPetSet(zookeeperManifestPath, ns)
+	return z.ps
+}
+
+func (z *zookeeperTester) write(petIndex int, kv map[string]string) {
+	name := fmt.Sprintf("%v-%d", z.ps.Name, petIndex)
+	ns := fmt.Sprintf("--namespace=%v", z.ps.Namespace)
+	for k, v := range kv {
+		cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh create /%v %v", k, v)
+		framework.Logf(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd))
+	}
+}
+
+func (z *zookeeperTester) read(petIndex int, key string) string {
+	name := fmt.Sprintf("%v-%d", z.ps.Name, petIndex)
+	ns := fmt.Sprintf("--namespace=%v", z.ps.Namespace)
+	cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh get /%v", key)
+	return lastLine(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd))
+}
+
+type mysqlGaleraTester struct {
+	ps     *apps.PetSet
+	tester *petSetTester
+}
+
+func (m *mysqlGaleraTester) name() string {
+	return "mysql: galera"
+}
+
+func (m *mysqlGaleraTester) mysqlExec(cmd, ns, podName string) string {
+	cmd = fmt.Sprintf("/usr/bin/mysql -u root -B -e '%v'", cmd)
+	// TODO: Find a readiness probe for mysql that guarantees writes will
+	// succeed and ditch retries. Current probe only reads, so there's a window
+	// for a race.
+	return kubectlExecWithRetries(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd)
+}
+
+func (m *mysqlGaleraTester) deploy(ns string) *apps.PetSet {
+	m.ps = m.tester.createPetSet(mysqlGaleraManifestPath, ns)
+
+	framework.Logf("Deployed petset %v, initializing database", m.ps.Name)
+	for _, cmd := range []string{
+		"create database petset;",
+		"use petset; create table pet (k varchar(20), v varchar(20));",
+	} {
+		framework.Logf(m.mysqlExec(cmd, ns, fmt.Sprintf("%v-0", m.ps.Name)))
+	}
+	return m.ps
+}
+
+func (m *mysqlGaleraTester) write(petIndex int, kv map[string]string) {
+	name := fmt.Sprintf("%v-%d", m.ps.Name, petIndex)
+	for k, v := range kv {
+		cmd := fmt.Sprintf("use  petset; insert into pet (k, v) values (\"%v\", \"%v\");", k, v)
+		framework.Logf(m.mysqlExec(cmd, m.ps.Namespace, name))
+	}
+}
+
+func (m *mysqlGaleraTester) read(petIndex int, key string) string {
+	name := fmt.Sprintf("%v-%d", m.ps.Name, petIndex)
+	return lastLine(m.mysqlExec(fmt.Sprintf("use petset; select v from pet where k=\"%v\";", key), m.ps.Namespace, name))
+}
+
+type redisTester struct {
+	ps     *apps.PetSet
+	tester *petSetTester
+}
+
+func (m *redisTester) name() string {
+	return "redis: master/slave"
+}
+
+func (m *redisTester) redisExec(cmd, ns, podName string) string {
+	cmd = fmt.Sprintf("/opt/redis/redis-cli -h %v %v", podName, cmd)
+	return framework.RunKubectlOrDie(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd)
+}
+
+func (m *redisTester) deploy(ns string) *apps.PetSet {
+	m.ps = m.tester.createPetSet(redisManifestPath, ns)
+	return m.ps
+}
+
+func (m *redisTester) write(petIndex int, kv map[string]string) {
+	name := fmt.Sprintf("%v-%d", m.ps.Name, petIndex)
+	for k, v := range kv {
+		framework.Logf(m.redisExec(fmt.Sprintf("SET %v %v", k, v), m.ps.Namespace, name))
+	}
+}
+
+func (m *redisTester) read(petIndex int, key string) string {
+	name := fmt.Sprintf("%v-%d", m.ps.Name, petIndex)
+	return lastLine(m.redisExec(fmt.Sprintf("GET %v", key), m.ps.Namespace, name))
+}
+
+func lastLine(out string) string {
+	outLines := strings.Split(strings.Trim(out, "\n"), "\n")
+	return outLines[len(outLines)-1]
+}
+
+func petSetFromManifest(fileName, ns string) *apps.PetSet {
+	var ps apps.PetSet
+	framework.Logf("Parsing petset from %v", fileName)
+	data, err := ioutil.ReadFile(fileName)
+	Expect(err).NotTo(HaveOccurred())
+	json, err := utilyaml.ToJSON(data)
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &ps)).NotTo(HaveOccurred())
+	ps.Namespace = ns
+	if ps.Spec.Selector == nil {
+		ps.Spec.Selector = &unversioned.LabelSelector{
+			MatchLabels: ps.Spec.Template.Labels,
+		}
+	}
+	return &ps
+}
+
+type petSetTester struct {
 	c *client.Client
 }
 
-func (p *petTester) execInPets(ps *apps.PetSet, cmd string) {
+func (p *petSetTester) createPetSet(manifestPath, ns string) *apps.PetSet {
+	mkpath := func(file string) string {
+		return filepath.Join(framework.TestContext.RepoRoot, manifestPath, file)
+	}
+	ps := petSetFromManifest(mkpath("petset.yaml"), ns)
+
+	framework.Logf(fmt.Sprintf("creating " + ps.Name + " service"))
+	framework.RunKubectlOrDie("create", "-f", mkpath("service.yaml"), fmt.Sprintf("--namespace=%v", ns))
+
+	framework.Logf(fmt.Sprintf("creating petset %v/%v with %d replicas and selector %+v", ps.Namespace, ps.Name, ps.Spec.Replicas, ps.Spec.Selector))
+	framework.RunKubectlOrDie("create", "-f", mkpath("petset.yaml"), fmt.Sprintf("--namespace=%v", ns))
+	p.waitForRunning(ps.Spec.Replicas, ps)
+	return ps
+}
+
+func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) {
 	podList := p.getPodList(ps)
 	for _, pet := range podList.Items {
 		stdout, err := framework.RunHostCmd(pet.Namespace, pet.Name, cmd)
 		ExpectNoError(err)
-		framework.Logf("stdout %v on %v: %v", cmd, pet.Name, stdout)
+		framework.Logf("stdout of %v on %v: %v", cmd, pet.Name, stdout)
 	}
 }
 
-func (p *petTester) saturate(ps *apps.PetSet) {
+func (p *petSetTester) saturate(ps *apps.PetSet) {
 	// TOOD: Watch events and check that creation timestamps don't overlap
 	for i := 0; i < ps.Spec.Replicas; i++ {
 		framework.Logf("Waiting for pet at index " + fmt.Sprintf("%v", i+1) + " to enter Running")
@@ -158,7 +421,7 @@ func (p *petTester) saturate(ps *apps.PetSet) {
 	}
 }
 
-func (p *petTester) deletePetAtIndex(index int, ps *apps.PetSet) {
+func (p *petSetTester) deletePetAtIndex(index int, ps *apps.PetSet) {
 	// TODO: we won't use "-index" as the name strategy forever,
 	// pull the name out from an identity mapper.
 	name := fmt.Sprintf("%v-%v", ps.Name, index)
@@ -168,7 +431,7 @@ func (p *petTester) deletePetAtIndex(index int, ps *apps.PetSet) {
 	}
 }
 
-func (p *petTester) restart(ps *apps.PetSet) {
+func (p *petSetTester) restart(ps *apps.PetSet) {
 	name := ps.Name
 	ns := ps.Namespace
 	oldReplicas := ps.Spec.Replicas
@@ -194,7 +457,7 @@ func (p *petTester) restart(ps *apps.PetSet) {
 	p.update(ns, name, func(ps *apps.PetSet) { ps.Spec.Replicas = oldReplicas })
 }
 
-func (p *petTester) update(ns, name string, update func(ps *apps.PetSet)) {
+func (p *petSetTester) update(ns, name string, update func(ps *apps.PetSet)) {
 	for i := 0; i < 3; i++ {
 		ps, err := p.c.Apps().PetSets(ns).Get(name)
 		if err != nil {
@@ -212,7 +475,7 @@ func (p *petTester) update(ns, name string, update func(ps *apps.PetSet)) {
 	framework.Failf("too many retries draining petset %q", name)
 }
 
-func (p *petTester) getPodList(ps *apps.PetSet) *api.PodList {
+func (p *petSetTester) getPodList(ps *apps.PetSet) *api.PodList {
 	selector, err := unversioned.LabelSelectorAsSelector(ps.Spec.Selector)
 	ExpectNoError(err)
 	podList, err := p.c.Pods(ps.Namespace).List(api.ListOptions{LabelSelector: selector})
@@ -220,11 +483,7 @@ func (p *petTester) getPodList(ps *apps.PetSet) *api.PodList {
 	return podList
 }
 
-func ExpectNoError(err error) {
+func (p *petSetTester) confirmPetCount(count int, ps *apps.PetSet, timeout time.Duration) {
-	Expect(err).NotTo(HaveOccurred())
-}
-
-func (p *petTester) confirmPetCount(count int, ps *apps.PetSet, timeout time.Duration) {
 	start := time.Now()
 	deadline := start.Add(timeout)
 	for t := time.Now(); t.Before(deadline); t = time.Now() {
@@ -238,7 +497,7 @@ func (p *petTester) confirmPetCount(count int, ps *apps.PetSet, timeout time.Dur
 	}
 }
 
-func (p *petTester) waitForRunning(numPets int, ps *apps.PetSet) {
+func (p *petSetTester) waitForRunning(numPets int, ps *apps.PetSet) {
 	pollErr := wait.PollImmediate(petsetPoll, petsetTimeout,
 		func() (bool, error) {
 			podList := p.getPodList(ps)
@@ -247,11 +506,12 @@ func (p *petTester) waitForRunning(numPets int, ps *apps.PetSet) {
 				return false, nil
 			}
 			if len(podList.Items) > numPets {
-				return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numPods, len(podList.Items))
+				return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numPets, len(podList.Items))
 			}
 			for _, p := range podList.Items {
-				if p.Status.Phase != api.PodRunning {
+				isReady := api.IsPodReady(&p)
-					framework.Logf("Waiting for pod %v to enter %v, currently %v", p.Name, api.PodRunning, p.Status.Phase)
+				if p.Status.Phase != api.PodRunning || !isReady {
+					framework.Logf("Waiting for pod %v to enter %v - Ready=True, currently %v - Ready=%v", p.Name, api.PodRunning, p.Status.Phase, isReady)
 					return false, nil
 				}
 			}
@@ -262,7 +522,7 @@ func (p *petTester) waitForRunning(numPets int, ps *apps.PetSet) {
 	}
 }
 
-func (p *petTester) setHealthy(ps *apps.PetSet) {
+func (p *petSetTester) setHealthy(ps *apps.PetSet) {
 	podList := p.getPodList(ps)
 	markedHealthyPod := ""
 	for _, pod := range podList.Items {
@@ -284,6 +544,10 @@ func (p *petTester) setHealthy(ps *apps.PetSet) {
 	}
 }
 
+func ExpectNoError(err error) {
+	Expect(err).NotTo(HaveOccurred())
+}
+
 func isInitialized(pod api.Pod) bool {
 	initialized, ok := pod.Annotations[petset.PetSetInitAnnotation]
 	if !ok {

test/e2e/testing-manifests/petset/mysql-galera/petset.yaml

@@ -0,0 +1,107 @@
+apiVersion: apps/v1alpha1
+kind: PetSet
+metadata:
+  name: mysql
+spec:
+  serviceName: "galera"
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: mysql
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+        pod.alpha.kubernetes.io/init-containers: '[
+            {
+                "name": "install",
+                "image": "gcr.io/google_containers/galera-install:0.1",
+                "imagePullPolicy": "Always",
+                "args": ["--work-dir=/work-dir"],
+                "volumeMounts": [
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    },
+                    {
+                        "name": "config",
+                        "mountPath": "/etc/mysql"
+                    }
+                ]
+            },
+            {
+                "name": "bootstrap",
+                "image": "debian:jessie",
+                "command": ["/work-dir/peer-finder"],
+                "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=galera"],
+                "env": [
+                  {
+                      "name": "POD_NAMESPACE",
+                      "valueFrom": {
+                          "fieldRef": {
+                              "apiVersion": "v1",
+                              "fieldPath": "metadata.namespace"
+                          }
+                      }
+                   }
+                ],
+                "volumeMounts": [
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    },
+                    {
+                        "name": "config",
+                        "mountPath": "/etc/mysql"
+                    }
+                ]
+            }
+        ]'
+    spec:
+      containers:
+      - name: mysql
+        image: gcr.io/google_containers/mysql-galera:e2e
+        ports:
+        - containerPort: 3306
+          name: mysql
+        - containerPort: 4444
+          name: sst
+        - containerPort: 4567
+          name: replication
+        - containerPort: 4568
+          name: ist
+        args:
+        - --defaults-file=/etc/mysql/my-galera.cnf
+        - --user=root
+        readinessProbe:
+          # TODO: If docker exec is buggy just use nc mysql-id 3306 as a ping.
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 15
+          timeoutSeconds: 5
+          successThreshold: 2
+        volumeMounts:
+        - name: datadir
+          mountPath: /var/lib/
+        - name: config
+          mountPath: /etc/mysql
+      - name: healthz
+        image: gcr.io/google_containers/mysql-healthz:1.0
+        args:
+        - --port=8080
+        - --verbose=true
+      volumes:
+      - name: config
+        emptyDir: {}
+      - name: workdir
+        emptyDir: {}
+  volumeClaimTemplates:
+  - metadata:
+      name: datadir
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 10Gi

test/e2e/testing-manifests/petset/mysql-galera/service.yaml

@@ -0,0 +1,18 @@
+# A headless service to create DNS records
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+  name: galera
+  labels:
+    app: mysql
+spec:
+  ports:
+  - port: 3306
+    name: mysql
+  # *.galear.default.svc.cluster.local
+  clusterIP: None
+  selector:
+    app: mysql
+

test/e2e/testing-manifests/petset/redis/petset.yaml

@@ -0,0 +1,97 @@
+apiVersion: apps/v1alpha1
+kind: PetSet
+metadata:
+  name: rd
+spec:
+  serviceName: "redis"
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: redis
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+        pod.alpha.kubernetes.io/init-containers: '[
+            {
+                "name": "install",
+                "image": "gcr.io/google_containers/redis-install:0.1",
+                "imagePullPolicy": "Always",
+                "args": ["--version=3.2.0", "--install-into=/opt", "--work-dir=/work-dir"],
+                "volumeMounts": [
+                    {
+                        "name": "opt",
+                        "mountPath": "/opt"
+                    },
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    }
+                ]
+            },
+            {
+                "name": "bootstrap",
+                "image": "debian:jessie",
+                "command": ["/work-dir/peer-finder"],
+                "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=redis"],
+                "env": [
+                  {
+                      "name": "POD_NAMESPACE",
+                      "valueFrom": {
+                          "fieldRef": {
+                              "apiVersion": "v1",
+                              "fieldPath": "metadata.namespace"
+                          }
+                      }
+                   }
+                ],
+                "volumeMounts": [
+                    {
+                        "name": "opt",
+                        "mountPath": "/opt"
+                    },
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    }
+                ]
+            }
+        ]'
+    spec:
+      containers:
+      - name: redis
+        image: debian:jessie
+        ports:
+        - containerPort: 6379
+          name: peer
+        command:
+        - /opt/redis/redis-server
+        args:
+        - /opt/redis/redis.conf
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - "/opt/redis/redis-cli -h $(hostname) ping"
+          initialDelaySeconds: 15
+          timeoutSeconds: 5
+        volumeMounts:
+        - name: datadir
+          mountPath: /data
+        - name: opt
+          mountPath: /opt
+      volumes:
+      - name: opt
+        emptyDir: {}
+      - name: workdir
+        emptyDir: {}
+  volumeClaimTemplates:
+  - metadata:
+      name: datadir
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 10Gi

test/e2e/testing-manifests/petset/redis/service.yaml

@@ -0,0 +1,18 @@
+# A headless service to create DNS records
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+  name: redis
+  labels:
+    app: redis
+spec:
+  ports:
+  - port: 6379
+    name: peer
+  # *.redis.default.svc.cluster.local
+  clusterIP: None
+  selector:
+    app: redis
+

test/e2e/testing-manifests/petset/zookeeper/petset.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1alpha1
+kind: PetSet
+metadata:
+  name: zoo
+spec:
+  serviceName: "zk"
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: zk
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+        pod.alpha.kubernetes.io/init-containers: '[
+            {
+                "name": "install",
+                "image": "gcr.io/google_containers/zookeeper-install:0.1",
+                "imagePullPolicy": "Always",
+                "args": ["--version=3.5.0-alpha", "--install-into=/opt", "--work-dir=/work-dir"],
+                "volumeMounts": [
+                    {
+                        "name": "opt",
+                        "mountPath": "/opt/"
+                    },
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    }
+                ]
+            },
+            {
+                "name": "bootstrap",
+                "image": "java:openjdk-8-jre",
+                "command": ["/work-dir/peer-finder"],
+                "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=zk"],
+                "env": [
+                  {
+                      "name": "POD_NAMESPACE",
+                      "valueFrom": {
+                          "fieldRef": {
+                              "apiVersion": "v1",
+                              "fieldPath": "metadata.namespace"
+                          }
+                      }
+                   }
+                ],
+                "volumeMounts": [
+                    {
+                        "name": "opt",
+                        "mountPath": "/opt/"
+                    },
+                    {
+                        "name": "workdir",
+                        "mountPath": "/work-dir"
+                    },
+                    {
+                        "name": "datadir",
+                        "mountPath": "/tmp/zookeeper"
+                    }
+                ]
+            }
+        ]'
+    spec:
+      containers:
+      - name: zk
+        image: java:openjdk-8-jre
+        ports:
+        - containerPort: 2888
+          name: peer
+        - containerPort: 3888
+          name: leader-election
+        command:
+        - /opt/zookeeper/bin/zkServer.sh
+        args:
+        - start-foreground
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - "/opt/zookeeper/bin/zkCli.sh ls /"
+          initialDelaySeconds: 15
+          timeoutSeconds: 5
+        volumeMounts:
+        - name: datadir
+          mountPath: /tmp/zookeeper
+        - name: opt
+          mountPath: /opt/
+      volumes:
+      - name: opt
+        emptyDir: {}
+      - name: workdir
+        emptyDir: {}
+  volumeClaimTemplates:
+  - metadata:
+      name: datadir
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 10Gi

test/e2e/testing-manifests/petset/zookeeper/service.yaml

@@ -0,0 +1,20 @@
+# A headless service to create DNS records
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+  name: zk
+  labels:
+    app: zk
+spec:
+  ports:
+  - port: 2888
+    name: peer
+  - port: 3888
+    name: leader-election
+  # *.zk.default.svc.cluster.local
+  clusterIP: None
+  selector:
+    app: zk
+