From fe0741bffe2528761ac68ca47583abc1321eef93 Mon Sep 17 00:00:00 2001
From: Jan Safranek <jsafrane@redhat.com>
Date: Thu, 29 Oct 2015 11:03:34 +0100
Subject: [PATCH] Configure cluster for e2e tests.

When KUBE_E2E_STORAGE_TEST_ENVIRONMENT is set to 'true', kube-up.sh script
will:

- Install the right packages for all storage volumes.
- Use devicemapper as docker storage backend. 'aufs', the default one on
Debian, does not support extended attibutes required by Ceph RBD and Gluster
server containers.

Tested on GCE and Vagrant, e2e tests for storage volumes passes without any
additional configuration.
---
 cluster/aws/config-default.sh                 |  3 ++
 cluster/aws/config-test.sh                    |  3 ++
 .../templates/create-dynamic-salt-files.sh    |  1 +
 cluster/aws/util.sh                           |  1 +
 cluster/gce/config-default.sh                 |  3 ++
 cluster/gce/config-test.sh                    |  3 ++
 cluster/gce/configure-vm.sh                   |  1 +
 cluster/gce/debian/helper.sh                  |  1 +
 cluster/saltbase/salt/base.sls                |  2 -
 cluster/saltbase/salt/docker/docker-defaults  |  6 ++-
 cluster/saltbase/salt/e2e/init.sls            | 40 +++++++++++++++++++
 cluster/saltbase/salt/top.sls                 |  3 ++
 cluster/vagrant/config-default.sh             |  2 +
 cluster/vagrant/config-test.sh                |  3 ++
 cluster/vagrant/provision-master.sh           |  1 +
 cluster/vagrant/util.sh                       |  2 +
 cluster/vsphere/config-default.sh             |  3 ++
 cluster/vsphere/config-test.sh                |  3 ++
 .../templates/create-dynamic-salt-files.sh    |  2 +
 cluster/vsphere/util.sh                       |  1 +
 20 files changed, 81 insertions(+), 3 deletions(-)
 create mode 100644 cluster/saltbase/salt/e2e/init.sls

diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh
index 79037c59de2..1abbf3a18cc 100644
--- a/cluster/aws/config-default.sh
+++ b/cluster/aws/config-default.sh
@@ -145,3 +145,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh
index 63e8519bd1e..04e36bfed80 100755
--- a/cluster/aws/config-test.sh
+++ b/cluster/aws/config-test.sh
@@ -137,3 +137,6 @@ KUBE_MINION_IMAGE="${KUBE_MINION_IMAGE:-}"
 COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}"
 CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}"
 RKT_VERSION="${KUBE_RKT_VERSION:-0.5.5}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/aws/templates/create-dynamic-salt-files.sh b/cluster/aws/templates/create-dynamic-salt-files.sh
index c74a2c48fcc..a2d03c5cc4a 100644
--- a/cluster/aws/templates/create-dynamic-salt-files.sh
+++ b/cluster/aws/templates/create-dynamic-salt-files.sh
@@ -41,6 +41,7 @@ opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG")'
 opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG")'
 opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET")'
 num_nodes: $(echo "${NUM_MINIONS}")
+e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF
 
 if [ -n "${ENABLE_EXPERIMENTAL_API:-}" ]; then
diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh
index 16a1a5cd7d4..193002127af 100755
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@@ -839,6 +839,7 @@ function kube-up {
     echo "readonly OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'"
     echo "readonly OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'"
     echo "readonly OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
+    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/setup-master-pd.sh"
diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh
index d5d18e7abdc..d8c8d4a2e58 100755
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@@ -119,3 +119,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh
index 32cab384e8e..d63bc62d299 100755
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@@ -134,3 +134,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh
index e1055c8b181..5328ecde0ec 100755
--- a/cluster/gce/configure-vm.sh
+++ b/cluster/gce/configure-vm.sh
@@ -285,6 +285,7 @@ enable_manifest_url: '$(echo "$ENABLE_MANIFEST_URL" | sed -e "s/'/''/g")'
 manifest_url: '$(echo "$MANIFEST_URL" | sed -e "s/'/''/g")'
 manifest_url_header: '$(echo "$MANIFEST_URL_HEADER" | sed -e "s/'/''/g")'
 num_nodes: $(echo "${NUM_MINIONS}")
+e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF
 
     if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh
index 82794f0c706..b7ff21358f9 100755
--- a/cluster/gce/debian/helper.sh
+++ b/cluster/gce/debian/helper.sh
@@ -60,6 +60,7 @@ NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
 OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
 OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
 OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
+E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
 EOF
   if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
     cat >>$file <<EOF
diff --git a/cluster/saltbase/salt/base.sls b/cluster/saltbase/salt/base.sls
index 196f1cf6db1..04cc1a03e2b 100644
--- a/cluster/saltbase/salt/base.sls
+++ b/cluster/saltbase/salt/base.sls
@@ -5,11 +5,9 @@ pkg-core:
 {% if grains['os_family'] == 'RedHat' %}
       - python
       - git
-      - glusterfs-fuse
 {% else %}
       - apt-transport-https
       - python-apt
-      - glusterfs-client
       - nfs-common
       - socat
 {% endif %}
diff --git a/cluster/saltbase/salt/docker/docker-defaults b/cluster/saltbase/salt/docker/docker-defaults
index 5c73cc61dbd..9ea5a121417 100644
--- a/cluster/saltbase/salt/docker/docker-defaults
+++ b/cluster/saltbase/salt/docker/docker-defaults
@@ -1,3 +1,7 @@
 {% set grains_opts = grains.get('docker_opts', '') -%}
-DOCKER_OPTS="{{grains_opts}} --bridge=cbr0 --iptables=false --ip-masq=false --log-level=warn"
+{% set e2e_opts = '' -%}
+{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' -%}
+{% set e2e_opts = '-s devicemapper' -%}
+{% endif -%}
+DOCKER_OPTS="{{grains_opts}} {{e2e_opts}} --bridge=cbr0 --iptables=false --ip-masq=false --log-level=warn"
 DOCKER_NOFILE=1000000
diff --git a/cluster/saltbase/salt/e2e/init.sls b/cluster/saltbase/salt/e2e/init.sls
new file mode 100644
index 00000000000..ca59041ea66
--- /dev/null
+++ b/cluster/saltbase/salt/e2e/init.sls
@@ -0,0 +1,40 @@
+e2e:
+
+{% if grains['os_family'] == 'Debian' and  grains['oscodename'] == 'wheezy' %}
+  # Add GlusterFS 3.5.2 Debian repo - Wheezy has Gluster 3.2 and that's too old for us.
+  pkgrepo.managed:
+    - name: deb http://download.gluster.org/pub/gluster/glusterfs/3.5/3.5.2/Debian/wheezy/apt wheezy main
+    - dist: wheezy
+    - file: /etc/apt/sources.list.d/gluster.list
+    - key_url: http://download.gluster.org/pub/gluster/glusterfs/3.5/3.5.2/Debian/wheezy/pubkey.gpg
+{% endif %}
+
+  # Install various packages required by e2e tests to all hosts.
+  # 'pkg.latest' is used to install updated glusterfs-client from the repo above,
+  # GCE image already has glusterfs-client 3.2.5, which is too old.
+  pkg.latest:
+    - refresh: true
+    - pkgs:
+      - targetcli
+      - ceph
+{% if grains['os_family'] == 'RedHat' %}
+      - glusterfs-fuse
+      - rbd-fuse
+      - iscsi-initiator-utils
+      - nfs-utils
+{% else %}
+      - glusterfs-client
+      - open-iscsi
+      - iscsitarget-dkms
+      - nfs-common
+{% endif %}
+
+
+
+{% if grains['os_family'] == 'Debian' %}
+# On Debian, re-start open-iscsi to generate unique
+# /etc/iscsi/initiatorname.iscsi
+open-iscsi:
+  cmd.run:
+    - name: 'service open-iscsi restart'
+{% endif %}
diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls
index ee3316275c8..b820b62a794 100644
--- a/cluster/saltbase/salt/top.sls
+++ b/cluster/saltbase/salt/top.sls
@@ -6,6 +6,9 @@ base:
 {% if grains.get('cloud') == 'aws' %}
     - ntp
 {% endif %}
+{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' %}
+    - e2e
+{% endif %}
 
   'roles:kubernetes-pool':
     - match: grain
diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh
index 65eff2adf79..be817012209 100755
--- a/cluster/vagrant/config-default.sh
+++ b/cluster/vagrant/config-default.sh
@@ -105,3 +105,5 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/vagrant/config-test.sh b/cluster/vagrant/config-test.sh
index 8d4ce6a76fa..8998b533905 100644
--- a/cluster/vagrant/config-test.sh
+++ b/cluster/vagrant/config-test.sh
@@ -23,3 +23,6 @@ source "${KUBE_ROOT}/cluster/vagrant/config-default.sh"
 
 # Do not register the master kubelet during testing
 REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh
index ebcd351b26e..da94a72cf72 100755
--- a/cluster/vagrant/provision-master.sh
+++ b/cluster/vagrant/provision-master.sh
@@ -132,6 +132,7 @@ cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
   opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG" | sed -e "s/'/''/g")'
   opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG" | sed -e "s/'/''/g")'
   opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET" | sed -e "s/'/''/g")'
+  e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF
 
 # Configure the salt-master
diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh
index aba505589b1..bf7e3b91d17 100644
--- a/cluster/vagrant/util.sh
+++ b/cluster/vagrant/util.sh
@@ -170,6 +170,7 @@ function create-provision-scripts {
     echo "OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'"
     echo "OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'"
     echo "OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
+    echo "E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
     awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-master.sh"
     awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-master.sh"
   ) > "${KUBE_TEMP}/master-start.sh"
@@ -195,6 +196,7 @@ function create-provision-scripts {
       echo "KUBELET_TOKEN='${KUBELET_TOKEN:-}'"
       echo "KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN:-}'"
       echo "MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
+      echo "E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
       awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-minion.sh"
       awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-minion.sh"
     ) > "${KUBE_TEMP}/minion-start-${i}.sh"
diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh
index 3cb73a2be58..e039f4b71f8 100755
--- a/cluster/vsphere/config-default.sh
+++ b/cluster/vsphere/config-default.sh
@@ -55,3 +55,6 @@ DNS_REPLICAS=1
 
 # Optional: Install Kubernetes UI
 ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/vsphere/config-test.sh b/cluster/vsphere/config-test.sh
index 57cbe8e935e..88740908d0d 100755
--- a/cluster/vsphere/config-test.sh
+++ b/cluster/vsphere/config-test.sh
@@ -32,3 +32,6 @@ MINION_MEMORY_MB=1024
 MINION_CPU=1
 
 SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"  # formerly PORTAL_NET
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/vsphere/templates/create-dynamic-salt-files.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh
index c9468cd4803..cfc8e9eba3d 100755
--- a/cluster/vsphere/templates/create-dynamic-salt-files.sh
+++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh
@@ -33,6 +33,8 @@ enable_cluster_dns: $ENABLE_CLUSTER_DNS
 dns_replicas: $DNS_REPLICAS
 dns_server: $DNS_SERVER_IP
 dns_domain: $DNS_DOMAIN
+e2e_storage_test_environment: $E2E_STORAGE_TEST_ENVIRONMENT
+
 EOF
 
 mkdir -p /srv/salt-overlay/salt/nginx
diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh
index e2ed4ca84ba..bd5e5f2eec4 100755
--- a/cluster/vsphere/util.sh
+++ b/cluster/vsphere/util.sh
@@ -250,6 +250,7 @@ function kube-up {
     echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
     echo "readonly SALT_TAR='${SALT_TAR##*/}'"
     echo "readonly MASTER_HTPASSWD='${htpasswd}'"
+    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-master.sh"