Merge pull request #7435 from jsafrane/devel/volume-tests

V2: Add e2e tests for NFS and Gluster

cluster/saltbase/salt/base.sls

@@ -5,7 +5,9 @@ pkg-core:
 {% if grains['os_family'] == 'RedHat' %}
       - python
       - git
+      - glusterfs-fuse
 {% else %}
       - apt-transport-https
       - python-apt
+      - glusterfs-client
 {% endif %}

contrib/for-tests/volumes-tester/gluster/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:14.04
+MAINTAINER Jan Safranek, jsafrane@redhat.com
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update -qq && apt-get install -y glusterfs-server -qq
+ADD glusterd.vol /etc/glusterfs/
+ADD run_gluster.sh /usr/local/bin/
+ADD index.html /vol/
+
+EXPOSE 24007/tcp 24008/tcp 49152/tcp
+
+ENTRYPOINT ["/usr/local/bin/run_gluster.sh"]

contrib/for-tests/volumes-tester/gluster/Makefile

@@ -0,0 +1,11 @@
+all: push
+
+TAG = 0.1
+
+container:
+	docker build -t gcr.io/google_containers/volume-gluster:$(TAG) .
+
+push: container
+	gcloud preview docker push gcr.io/google_containers/volume-gluster:$(TAG)
+
+clean:

contrib/for-tests/volumes-tester/gluster/README.md

@@ -0,0 +1,8 @@
+# Gluster server container for testing
+
+This container exports test_vol volume with an index.html inside.
+
+Used by test/e2e/* to test GlusterfsVolumeSource. Not for production use!
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/for-tests/volumes-tester/gluster/README.md?pixel)]()

contrib/for-tests/volumes-tester/gluster/glusterd.vol

@@ -0,0 +1,14 @@
+# This is default glusterd.vol (incl. commented out base-port),
+# with added "rpc-auth-allow-insecure on" to allow connection
+# from non-privileged ports.
+
+volume management
+    type mgmt/glusterd
+    option working-directory /var/lib/glusterd
+    option transport-type socket,rdma
+    option transport.socket.keepalive-time 10
+    option transport.socket.keepalive-interval 2
+    option transport.socket.read-fail-log off
+#   option base-port 49152
+    option rpc-auth-allow-insecure on
+end-volume

contrib/for-tests/volumes-tester/gluster/index.html

@@ -0,0 +1 @@
+Hello from GlusterFS!

contrib/for-tests/volumes-tester/gluster/run_gluster.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function start()
+{
+    /usr/sbin/glusterd -p /run/glusterd.pid
+    gluster volume create test_vol `hostname -i`:/vol force
+    gluster volume start test_vol
+}
+
+function stop()
+{
+    gluster --mode=script volume stop test_vol force
+    kill $(cat /run/glusterd.pid)
+    exit 0
+}
+
+
+trap stop TERM
+
+start "$@"
+
+while true; do
+ read
+done
+

contrib/for-tests/volumes-tester/nfs/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:14.04
+MAINTAINER Jan Safranek, jsafrane@redhat.com
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update -qq && apt-get install -y nfs-kernel-server -qq
+RUN mkdir -p /exports
+ADD run_nfs.sh /usr/local/bin/
+ADD index.html /exports/index.html
+
+EXPOSE 2049/tcp
+
+ENTRYPOINT ["/usr/local/bin/run_nfs.sh", "/exports"]

contrib/for-tests/volumes-tester/nfs/Makefile

@@ -0,0 +1,11 @@
+all: push
+
+TAG = 0.1
+
+container:
+	docker build -t gcr.io/google_containers/volume-nfs:$(TAG) .
+
+push: container
+	gcloud preview docker push gcr.io/google_containers/volume-nfs:$(TAG)
+
+clean:

contrib/for-tests/volumes-tester/nfs/README.md

@@ -0,0 +1,10 @@
+# NFS server container for testing
+
+This container exports '/' directory with an index.html inside. NFSv4 only.
+
+Inspired by https://github.com/cpuguy83/docker-nfs-server.
+
+Used by test/e2e/* to test NFSVolumeSource. Not for production use!
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/for-tests/volumes-tester/nfs/README.md?pixel)]()

contrib/for-tests/volumes-tester/nfs/index.html

@@ -0,0 +1 @@
+Hello from NFS!

contrib/for-tests/volumes-tester/nfs/run_nfs.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function start()
+{
+
+    # prepare /etc/exports
+    for i in "$@"; do
+        # fsid=0: needed for NFSv4
+        echo "$i *(rw,fsid=0)" >> /etc/exports
+        echo "Serving $i"
+    done
+
+    mount -t nfsd nfds /proc/fs/nfsd
+
+    # -N 2 -N 3: disable NFSv2+3
+    # -V 4.x: enable NFSv4
+    /usr/sbin/rpc.mountd -N 2 -N 3 -V 4 -V 4.1
+
+    /usr/sbin/exportfs -r
+    /usr/sbin/rpc.nfsd -N 2 -N 3 -V 4 -V 4.1 2
+
+    echo "NFS started"
+}
+
+function stop()
+{
+    echo "Stopping NFS"
+
+    /usr/sbin/rpc.nfsd 0
+    /usr/sbin/exportfs -au
+    /usr/sbin/exportfs -f
+
+    kill $( pidof rpc.mountd )
+    umount /proc/fs/nfsd
+    echo > /etc/exports
+    exit 0
+}
+
+
+trap stop TERM
+
+start "$@"
+
+# Ugly hack to do nothing and wait for SIGTERM
+while true; do
+    read
+done

examples/nfs/exporter/run_nfs

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2015 Red Hat Inc.
+# Copyright 2015 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

test/e2e/volumes.go

@@ -0,0 +1,332 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This test checks that various VolumeSources are working. For each volume
+ * type it creates a server pod, exporting simple 'index.html' file.
+ * Then it uses appropriate VolumeSource to import this file into a client pod
+ * and checks that the pod can see the file. It does so by importing the file
+ * into web server root and loadind the index.html from it.
+ *
+ * These tests work only when privileged containers are allowed, exporting
+ * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or
+ * other privileged magic in the server pod.
+ *
+ * Note that the server containers are for testing purposes only and should not
+ * be used in production.
+ */
+
+package e2e
+
+import (
+	"fmt"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Configuration of one tests. The test consist of:
+// - server pod - runs serverImage, exports ports[]
+// - client pod - does not need any special configuration
+type VolumeTestConfig struct {
+	namespace string
+	// Prefix of all pods. Typically the test name.
+	prefix string
+	// Name of container image for the server pod.
+	serverImage string
+	// Ports to export from the server pod. TCP only.
+	serverPorts []int
+}
+
+// Starts a container specified by config.serverImage and exports all
+// config.serverPorts from it. The returned pod should be used to get the server
+// IP address and create appropriate VolumeSource.
+func startVolumeServer(client *client.Client, config VolumeTestConfig) *api.Pod {
+	podClient := client.Pods(config.namespace)
+
+	portCount := len(config.serverPorts)
+	serverPodPorts := make([]api.ContainerPort, portCount)
+
+	for i := 0; i < portCount; i++ {
+		portName := fmt.Sprintf("%s-%d", config.prefix, i)
+
+		serverPodPorts[i] = api.ContainerPort{
+			Name:          portName,
+			ContainerPort: config.serverPorts[i],
+			Protocol:      api.ProtocolTCP,
+		}
+	}
+
+	By(fmt.Sprint("creating ", config.prefix, " server pod"))
+	privileged := new(bool)
+	*privileged = true
+	serverPod := &api.Pod{
+		TypeMeta: api.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1beta3",
+		},
+		ObjectMeta: api.ObjectMeta{
+			Name: config.prefix + "-server",
+			Labels: map[string]string{
+				"role": config.prefix + "-server",
+			},
+		},
+
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:  config.prefix + "-server",
+					Image: config.serverImage,
+					SecurityContext: &api.SecurityContext{
+						Privileged: privileged,
+					},
+					Ports: serverPodPorts,
+				},
+			},
+		},
+	}
+	_, err := podClient.Create(serverPod)
+	expectNoError(err, "Failed to create %s pod: %v", serverPod.Name, err)
+
+	expectNoError(waitForPodRunningInNamespace(client, serverPod.Name, config.namespace))
+
+	By("locating the server pod")
+	pod, err := podClient.Get(serverPod.Name)
+	expectNoError(err, "Cannot locate the server pod %v: %v", serverPod.Name, err)
+
+	By("sleeping a bit to give the server time to start")
+	time.Sleep(20 * time.Second)
+	return pod
+}
+
+// Clean both server and client pods.
+func volumeTestCleanup(client *client.Client, config VolumeTestConfig) {
+	By(fmt.Sprint("cleaning the environment after ", config.prefix))
+
+	defer GinkgoRecover()
+
+	podClient := client.Pods(config.namespace)
+
+	// ignore all errors, the pods may not be even created
+	podClient.Delete(config.prefix+"-client", nil)
+	podClient.Delete(config.prefix+"-server", nil)
+}
+
+// Start a client pod using given VolumeSource (exported by startVolumeServer())
+// and check that the pod sees the data from the server pod.
+func testVolumeClient(client *client.Client, config VolumeTestConfig, volume api.VolumeSource, expectedContent string) {
+	By(fmt.Sprint("starting ", config.prefix, " client"))
+	podClient := client.Pods(config.namespace)
+
+	clientPod := &api.Pod{
+		TypeMeta: api.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1beta3",
+		},
+		ObjectMeta: api.ObjectMeta{
+			Name: config.prefix + "-client",
+			Labels: map[string]string{
+				"role": config.prefix + "-client",
+			},
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:  config.prefix + "-client",
+					Image: "gcr.io/google_containers/nginx:1.7.9",
+					Ports: []api.ContainerPort{
+						{
+							Name:          "web",
+							ContainerPort: 80,
+							Protocol:      api.ProtocolTCP,
+						},
+					},
+					VolumeMounts: []api.VolumeMount{
+						{
+							Name:      config.prefix + "-volume",
+							MountPath: "/usr/share/nginx/html",
+						},
+					},
+				},
+			},
+			Volumes: []api.Volume{
+				{
+					Name:         config.prefix + "-volume",
+					VolumeSource: volume,
+				},
+			},
+		},
+	}
+	if _, err := podClient.Create(clientPod); err != nil {
+		Failf("Failed to create %s pod: %v", clientPod.Name, err)
+	}
+	expectNoError(waitForPodRunningInNamespace(client, clientPod.Name, config.namespace))
+
+	By("reading a web page from the client")
+	body, err := client.Get().
+		Namespace(config.namespace).
+		Prefix("proxy").
+		Resource("pods").
+		Name(clientPod.Name).
+		DoRaw()
+	expectNoError(err, "Cannot read web page: %v", err)
+	Logf("body: %v", string(body))
+
+	By("checking the page content")
+	Expect(body).To(ContainSubstring(expectedContent))
+}
+
+var _ = Describe("Volume", func() {
+	clean := true // If 'false', the test won't clear its namespace (and pods and services) upon completion. Useful for debugging.
+
+	// filled in BeforeEach
+	var c *client.Client
+	var namespace *api.Namespace
+
+	BeforeEach(func() {
+		var err error
+		c, err = loadClient()
+		Expect(err).NotTo(HaveOccurred())
+		By("Building a namespace api object")
+		namespace, err = createTestingNS("volume", c)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		if clean {
+			if err := c.Namespaces().Delete(namespace.Name); err != nil {
+				Failf("Couldn't delete ns %s", err)
+			}
+		}
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// NFS
+	////////////////////////////////////////////////////////////////////////
+
+	// Marked with [Skipped] to skip the test by default (see driver.go),
+	// the test needs privileged containers, which are disabled by default.
+	// Run the test with "go run hack/e2e.go ... --ginkgo.focus=Volume"
+	Describe("[Skipped] NFS", func() {
+		It("should be montable", func() {
+			config := VolumeTestConfig{
+				namespace:   namespace.Name,
+				prefix:      "nfs",
+				serverImage: "gcr.io/google_containers/volume-nfs",
+				serverPorts: []int{2049},
+			}
+
+			defer func() {
+				if clean {
+					volumeTestCleanup(c, config)
+				}
+			}()
+			pod := startVolumeServer(c, config)
+			serverIP := pod.Status.PodIP
+			Logf("NFS server IP address: %v", serverIP)
+
+			volume := api.VolumeSource{
+				NFS: &api.NFSVolumeSource{
+					Server:   serverIP,
+					Path:     "/",
+					ReadOnly: true,
+				},
+			}
+			// Must match content of contrib/for-tests/volumes-tester/nfs/index.html
+			testVolumeClient(c, config, volume, "Hello from NFS!")
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Gluster
+	////////////////////////////////////////////////////////////////////////
+
+	// Marked with [Skipped] to skip the test by default (see driver.go),
+	// the test needs privileged containers, which are disabled by default.
+	// Run the test with "go run hack/e2e.go ... --ginkgo.focus=Volume"
+	Describe("[Skipped] GlusterFS", func() {
+		It("should be mountable", func() {
+			config := VolumeTestConfig{
+				namespace:   namespace.Name,
+				prefix:      "gluster",
+				serverImage: "gcr.io/google_containers/volume-gluster",
+				serverPorts: []int{24007, 24008, 49152},
+			}
+
+			defer func() {
+				if clean {
+					volumeTestCleanup(c, config)
+				}
+			}()
+			pod := startVolumeServer(c, config)
+			serverIP := pod.Status.PodIP
+			Logf("Gluster server IP address: %v", serverIP)
+
+			// create Endpoints for the server
+			endpoints := api.Endpoints{
+				TypeMeta: api.TypeMeta{
+					Kind:       "Endpoints",
+					APIVersion: "v1beta3",
+				},
+				ObjectMeta: api.ObjectMeta{
+					Name: config.prefix + "-server",
+				},
+				Subsets: []api.EndpointSubset{
+					{
+						Addresses: []api.EndpointAddress{
+							{
+								IP: serverIP,
+							},
+						},
+						Ports: []api.EndpointPort{
+							{
+								Name:     "gluster",
+								Port:     24007,
+								Protocol: api.ProtocolTCP,
+							},
+						},
+					},
+				},
+			}
+
+			endClient := c.Endpoints(config.namespace)
+
+			defer func() {
+				if clean {
+					endClient.Delete(config.prefix + "-server")
+				}
+			}()
+
+			if _, err := endClient.Create(&endpoints); err != nil {
+				Failf("Failed to create endpoints for Gluster server: %v", err)
+			}
+
+			volume := api.VolumeSource{
+				Glusterfs: &api.GlusterfsVolumeSource{
+					EndpointsName: config.prefix + "-server",
+					// 'test_vol' comes from contrib/for-tests/volumes-tester/gluster/run_gluster.sh
+					Path:     "test_vol",
+					ReadOnly: true,
+				},
+			}
+			// Must match content of contrib/for-tests/volumes-tester/gluster/index.html
+			testVolumeClient(c, config, volume, "Hello from GlusterFS!")
+		})
+	})
+})