diff --git a/test/e2e/testing-manifests/statefulset/etcd/pdb.yaml b/test/e2e/testing-manifests/statefulset/etcd/pdb.yaml new file mode 100644 index 00000000000..64d68aac7c0 --- /dev/null +++ b/test/e2e/testing-manifests/statefulset/etcd/pdb.yaml @@ -0,0 +1,11 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: etcd-pdb + labels: + pdb: etcd +spec: + minAvailable: 2 + selector: + matchLabels: + app: etcd diff --git a/test/e2e/testing-manifests/statefulset/etcd/service.yaml b/test/e2e/testing-manifests/statefulset/etcd/service.yaml new file mode 100644 index 00000000000..3932f04f42b --- /dev/null +++ b/test/e2e/testing-manifests/statefulset/etcd/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +metadata: + name: etcd + labels: + app: etcd +spec: + ports: + - port: 2380 + name: etcd-server + - port: 2379 + name: etcd-client + clusterIP: None + selector: + app: etcd diff --git a/test/e2e/testing-manifests/statefulset/etcd/statefulset.yaml b/test/e2e/testing-manifests/statefulset/etcd/statefulset.yaml new file mode 100644 index 00000000000..c7800cd4e8f --- /dev/null +++ b/test/e2e/testing-manifests/statefulset/etcd/statefulset.yaml @@ -0,0 +1,178 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: etcd + labels: + app: etcd +spec: + serviceName: etcd + replicas: 3 + template: + metadata: + name: etcd + labels: + app: etcd + spec: + containers: + - name: etcd + image: gcr.io/google_containers/etcd-amd64:2.2.5 + imagePullPolicy: Always + ports: + - containerPort: 2380 + name: peer + - containerPort: 2379 + name: client + resources: + requests: + cpu: 100m + memory: 512Mi + env: + - name: INITIAL_CLUSTER_SIZE + value: "3" + - name: SET_NAME + value: etcd + volumeMounts: + - name: datadir + mountPath: /var/run/etcd + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-ec" + - | + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" + done + + HOSTNAME=$(hostname) + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + echo "Removing ${HOSTNAME} from etcd cluster" + + ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash) + if [ $? -eq 0 ]; then + # Remove everything otherwise the cluster will no longer scale-up + rm -rf /var/run/etcd/* + fi + command: + - "/bin/sh" + - "-ec" + - | + HOSTNAME=$(hostname) + + # store member id into PVC for later member replacement + collect_member() { + while ! etcdctl member list &>/dev/null; do sleep 1; done + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id + exit 0 + } + + eps() { + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" + done + echo ${EPS} + } + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + # re-joining after failure? + if [ -e /var/run/etcd/default.etcd ]; then + echo "Re-joining etcd member" + member_id=$(cat /var/run/etcd/member_id) + + # re-join member + ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd + fi + + # etcd-SET_ID + SET_ID=${HOSTNAME:5:${#HOSTNAME}} + + # adding a new member to existing cluster (assuming all initial pods are available) + if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then + export ETCDCTL_ENDPOINT=$(eps) + + # member already added? + MEMBER_HASH=$(member_hash) + if [ -n "${MEMBER_HASH}" ]; then + # the member hash exists but for some reason etcd failed + # as the datadir has not be created, we can remove the member + # and retrieve new hash + etcdctl member remove ${MEMBER_HASH} + fi + + echo "Adding new member" + etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs + + if [ $? -ne 0 ]; then + echo "Exiting" + rm -f /var/run/etcd/new_member_envs + exit 1 + fi + + cat /var/run/etcd/new_member_envs + source /var/run/etcd/new_member_envs + + collect_member & + + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --initial-cluster ${ETCD_INITIAL_CLUSTER} \ + --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} + fi + + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + while true; do + echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up" + ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break + sleep 1s + done + done + + PEERS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380" + done + + collect_member & + + # join member + exec etcd --name ${HOSTNAME} \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster ${PEERS} \ + --initial-cluster-state new \ + --data-dir /var/run/etcd/default.etcd + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + # upstream recommended max is 700M + storage: 1Gi + diff --git a/test/e2e/testing-manifests/statefulset/etcd/tester.yaml b/test/e2e/testing-manifests/statefulset/etcd/tester.yaml new file mode 100644 index 00000000000..c5ea0b90c14 --- /dev/null +++ b/test/e2e/testing-manifests/statefulset/etcd/tester.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: etcd-test-server +spec: + replicas: 3 + template: + metadata: + labels: + app: test-server + spec: + containers: + - name: test-server + image: gcr.io/google-containers/etcd-statefulset-e2e-test:0.0 + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 2 + periodSeconds: 2 +