Add Ceph RBD e2e test.

The test assumes that all nodes have Ceph client utilities installed.
Ceph RBD container is hand crafted to be really minimal. It creates a new RBD
on startup, which can take up to several minutes on busy machines.

contrib/for-tests/volumes-tester/rbd/Dockerfile

@@ -0,0 +1,34 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CEPH all in one
+# Based on image by Ricardo Rocha, ricardo@catalyst.net.nz
+
+FROM fedora
+MAINTAINER Jan Safranek jsafrane@redhat.com
+
+# Base Packages
+RUN yum install -y wget ceph ceph-fuse strace && yum clean all
+
+# Get ports exposed
+EXPOSE 6789
+
+ADD ./bootstrap.sh /bootstrap.sh
+ADD ./mon.sh /mon.sh
+ADD ./osd.sh /osd.sh
+ADD ./ceph.conf.sh /ceph.conf.sh
+ADD ./keyring /var/lib/ceph/mon/keyring
+ADD ./block.tar.gz /
+
+CMD /bootstrap.sh

contrib/for-tests/volumes-tester/rbd/Makefile

@@ -0,0 +1,22 @@
+all: push
+
+TAG = 0.1
+
+container:
+	# Build new image and automatically tag it as latest
+	docker build -t gcr.io/google_containers/volume-rbd .
+	# Add the version tag to the latest image 
+	docker tag gcr.io/google_containers/volume-rbd gcr.io/google_containers/volume-rbd:$(TAG)
+
+block:
+	# Create block.tar.gz with ext2 block device with index.html inside.
+	# block.tar.gz is already available in git and users don't need to
+	# regenerate it, this target is here just for reference.
+	# Run as root!
+	./create_block.sh
+
+push: container
+	# Push image tagged as latest to repository
+	gcloud docker push gcr.io/google_containers/volume-rbd
+	# Push version tagged image to repository (since this image is already pushed it will simply create or update version tag)
+	gcloud docker push gcr.io/google_containers/volume-rbd:$(TAG)

contrib/for-tests/volumes-tester/rbd/bootstrap.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Bootstraps a CEPH server.
+# It creates two OSDs on local machine, creates RBD pool there
+# and imports 'block' device there.
+#
+# We must create fresh OSDs and filesystem here, because shipping it
+# in a container would increase the image by ~300MB.
+#
+
+
+# Create /etc/ceph/ceph.conf
+sh ./ceph.conf.sh `hostname -i`
+
+# Configure and start ceph-mon
+sh ./mon.sh `hostname -i`
+
+# Configure and start 2x ceph-osd
+mkdir -p /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-1
+sh ./osd.sh 0
+sh ./osd.sh 1
+
+# Prepare a RBD volume
+# NOTE: we need Ceph kernel modules on the host!
+rbd import block foo
+
+echo "Ceph is ready"
+
+# Wait forever
+while true; do
+    sleep 10
+done

contrib/for-tests/volumes-tester/rbd/ceph.conf.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Configures /etc/ceph.conf from a template.
+#
+
+echo "
+[global]
+auth cluster required = none
+auth service required = none
+auth client required = none
+
+[mon.a]
+host = cephbox
+mon addr = $1
+
+[osd]
+osd journal size = 128 
+journal dio = false
+
+[osd.0]
+osd host = cephbox
+" > /etc/ceph/ceph.conf
+

contrib/for-tests/volumes-tester/rbd/create_block.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Create block.tar.gz with a small ext2 filesystem.
+# It must be run as root (to mount with '-o loop')!
+
+# Exit on the first error.
+set -e
+
+MNTDIR=`mktemp -d`
+
+cleanup()
+{
+    # Make sure we return the right exit code
+    RET=$?
+    # Silently remove everything and ignore errors
+    set +e
+    /bin/umount $MNTDIR 2>/dev/null
+    /bin/rmdir $MNTDIR 2>/dev/null
+    /bin/rm block 2>/dev/null
+    exit $RET
+}
+
+trap cleanup TERM EXIT
+
+# Create 1MB device with ext2
+dd if=/dev/zero of=block count=1 bs=1M
+mkfs.ext2 block
+
+# Add index.html to it
+mount -o loop block $MNTDIR
+echo "Hello from RBD" > $MNTDIR/index.html
+umount $MNTDIR
+
+rm block.tar.gz 2>/dev/null || :
+tar cfz block.tar.gz block

contrib/for-tests/volumes-tester/rbd/keyring

@@ -0,0 +1,8 @@
+[mon.]
+        key = AQDRrKNV6z4UChAABzP1ZyysTU4pjgjNOf/p3A==
+[client.admin]
+        key = AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA==
+        auid = 0
+        caps mds = "allow *"
+        caps mon = "allow *"
+        caps osd = "allow *"

contrib/for-tests/volumes-tester/rbd/mon.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Configures and launches a new MON.
+#
+
+# monitor setup
+monmaptool --create --clobber --fsid `uuidgen` --add a $1:6789 /etc/ceph/monmap
+mkdir /var/lib/ceph/mon/ceph-a
+ceph-mon -i a --mkfs --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/keyring
+cp /var/lib/ceph/mon/keyring /var/lib/ceph/mon/ceph-a
+ceph-mon -i a --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring
+
+# client setup (handy)
+cp /var/lib/ceph/mon/keyring /etc/ceph
+
+# for this test we want to
+ceph osd getcrushmap -o /tmp/crushc
+crushtool -d /tmp/crushc -o /tmp/crushd
+sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /tmp/crushd
+crushtool -c /tmp/crushd -o /tmp/crushc
+ceph osd setcrushmap -i /tmp/crushc

contrib/for-tests/volumes-tester/rbd/osd.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Configures and launches a new OSD.
+#
+
+ceph osd create
+ceph-osd -i $1 --mkfs --mkkey
+ceph auth add osd.$1 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-$1/keyring
+ceph osd crush add $1 1 root=default host=cephbox
+ceph-osd -i $1 -k /var/lib/ceph/osd/ceph-$1/keyring

test/e2e/volumes.go

@@ -400,4 +400,83 @@ var _ = Describe("Volumes", func() {
 			testVolumeClient(c, config, volume, "Hello from iSCSI")
 		})
 	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Ceph RBD
+	////////////////////////////////////////////////////////////////////////
+
+	// Marked with [Skipped] to skip the test by default (see driver.go),
+	// the test needs privileged containers, which are disabled by default.
+	// Run the test with "go run hack/e2e.go ... --ginkgo.focus=RBD"
+
+	// Run the test with "go run hack/e2e.go ... --ginkgo.focus=Volume"
+	Describe("[Skipped] Ceph RBD", func() {
+		It("should be mountable", func() {
+			config := VolumeTestConfig{
+				namespace:   namespace.Name,
+				prefix:      "rbd",
+				serverImage: "gcr.io/google_containers/volume-rbd",
+				serverPorts: []int{6789},
+				volumes: map[string]string{
+					// iSCSI container needs to insert modules from the host
+					"/lib/modules": "/lib/modules",
+					"/sys":         "/sys",
+				},
+			}
+
+			defer func() {
+				if clean {
+					volumeTestCleanup(c, config)
+				}
+			}()
+			pod := startVolumeServer(c, config)
+			serverIP := pod.Status.PodIP
+			Logf("Ceph server IP address: %v", serverIP)
+
+			// create secrets for the server
+			secret := api.Secret{
+				TypeMeta: api.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: api.ObjectMeta{
+					Name: config.prefix + "-secret",
+				},
+				Data: map[string][]byte{
+					// from contrib/for-tests/volumes-tester/rbd/keyring
+					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
+				},
+			}
+
+			secClient := c.Secrets(config.namespace)
+
+			defer func() {
+				if clean {
+					secClient.Delete(config.prefix + "-secret")
+				}
+			}()
+
+			if _, err := secClient.Create(&secret); err != nil {
+				Failf("Failed to create secrets for Ceph RBD: %v", err)
+			}
+
+			volume := api.VolumeSource{
+				RBD: &api.RBDVolumeSource{
+					CephMonitors: []string{serverIP},
+					RBDPool:      "rbd",
+					RBDImage:     "foo",
+					RadosUser:    "admin",
+					SecretRef: &api.LocalObjectReference{
+						Name: config.prefix + "-secret",
+					},
+					FSType:   "ext2",
+					ReadOnly: true,
+				},
+			}
+			// Must match content of contrib/for-tests/volumes-tester/gluster/index.html
+			testVolumeClient(c, config, volume, "Hello from RBD")
+
+		})
+	})
+
 })