diff --git a/contrib/for-tests/volumes-tester/rbd/Dockerfile b/contrib/for-tests/volumes-tester/rbd/Dockerfile new file mode 100644 index 00000000000..2a6498057da --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CEPH all in one +# Based on image by Ricardo Rocha, ricardo@catalyst.net.nz + +FROM fedora +MAINTAINER Jan Safranek jsafrane@redhat.com + +# Base Packages +RUN yum install -y wget ceph ceph-fuse strace && yum clean all + +# Get ports exposed +EXPOSE 6789 + +ADD ./bootstrap.sh /bootstrap.sh +ADD ./mon.sh /mon.sh +ADD ./osd.sh /osd.sh +ADD ./ceph.conf.sh /ceph.conf.sh +ADD ./keyring /var/lib/ceph/mon/keyring +ADD ./block.tar.gz / + +CMD /bootstrap.sh diff --git a/contrib/for-tests/volumes-tester/rbd/Makefile b/contrib/for-tests/volumes-tester/rbd/Makefile new file mode 100644 index 00000000000..a1d1e168828 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/Makefile @@ -0,0 +1,22 @@ +all: push + +TAG = 0.1 + +container: + # Build new image and automatically tag it as latest + docker build -t gcr.io/google_containers/volume-rbd . + # Add the version tag to the latest image + docker tag gcr.io/google_containers/volume-rbd gcr.io/google_containers/volume-rbd:$(TAG) + +block: + # Create block.tar.gz with ext2 block device with index.html inside. + # block.tar.gz is already available in git and users don't need to + # regenerate it, this target is here just for reference. + # Run as root! + ./create_block.sh + +push: container + # Push image tagged as latest to repository + gcloud docker push gcr.io/google_containers/volume-rbd + # Push version tagged image to repository (since this image is already pushed it will simply create or update version tag) + gcloud docker push gcr.io/google_containers/volume-rbd:$(TAG) diff --git a/contrib/for-tests/volumes-tester/rbd/block.tar.gz b/contrib/for-tests/volumes-tester/rbd/block.tar.gz new file mode 100644 index 00000000000..395d1492f9b Binary files /dev/null and b/contrib/for-tests/volumes-tester/rbd/block.tar.gz differ diff --git a/contrib/for-tests/volumes-tester/rbd/bootstrap.sh b/contrib/for-tests/volumes-tester/rbd/bootstrap.sh new file mode 100755 index 00000000000..f5ca9dd18d1 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/bootstrap.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Bootstraps a CEPH server. +# It creates two OSDs on local machine, creates RBD pool there +# and imports 'block' device there. +# +# We must create fresh OSDs and filesystem here, because shipping it +# in a container would increase the image by ~300MB. +# + + +# Create /etc/ceph/ceph.conf +sh ./ceph.conf.sh `hostname -i` + +# Configure and start ceph-mon +sh ./mon.sh `hostname -i` + +# Configure and start 2x ceph-osd +mkdir -p /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-1 +sh ./osd.sh 0 +sh ./osd.sh 1 + +# Prepare a RBD volume +# NOTE: we need Ceph kernel modules on the host! +rbd import block foo + +echo "Ceph is ready" + +# Wait forever +while true; do + sleep 10 +done \ No newline at end of file diff --git a/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh b/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh new file mode 100755 index 00000000000..f3f557585c5 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures /etc/ceph.conf from a template. +# + +echo " +[global] +auth cluster required = none +auth service required = none +auth client required = none + +[mon.a] +host = cephbox +mon addr = $1 + +[osd] +osd journal size = 128 +journal dio = false + +[osd.0] +osd host = cephbox +" > /etc/ceph/ceph.conf + diff --git a/contrib/for-tests/volumes-tester/rbd/create_block.sh b/contrib/for-tests/volumes-tester/rbd/create_block.sh new file mode 100755 index 00000000000..87c7eeaba74 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/create_block.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create block.tar.gz with a small ext2 filesystem. +# It must be run as root (to mount with '-o loop')! + +# Exit on the first error. +set -e + +MNTDIR=`mktemp -d` + +cleanup() +{ + # Make sure we return the right exit code + RET=$? + # Silently remove everything and ignore errors + set +e + /bin/umount $MNTDIR 2>/dev/null + /bin/rmdir $MNTDIR 2>/dev/null + /bin/rm block 2>/dev/null + exit $RET +} + +trap cleanup TERM EXIT + +# Create 1MB device with ext2 +dd if=/dev/zero of=block count=1 bs=1M +mkfs.ext2 block + +# Add index.html to it +mount -o loop block $MNTDIR +echo "Hello from RBD" > $MNTDIR/index.html +umount $MNTDIR + +rm block.tar.gz 2>/dev/null || : +tar cfz block.tar.gz block diff --git a/contrib/for-tests/volumes-tester/rbd/keyring b/contrib/for-tests/volumes-tester/rbd/keyring new file mode 100644 index 00000000000..f8c5af22417 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/keyring @@ -0,0 +1,8 @@ +[mon.] + key = AQDRrKNV6z4UChAABzP1ZyysTU4pjgjNOf/p3A== +[client.admin] + key = AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA== + auid = 0 + caps mds = "allow *" + caps mon = "allow *" + caps osd = "allow *" diff --git a/contrib/for-tests/volumes-tester/rbd/mon.sh b/contrib/for-tests/volumes-tester/rbd/mon.sh new file mode 100755 index 00000000000..8013044b89f --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/mon.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures and launches a new MON. +# + +# monitor setup +monmaptool --create --clobber --fsid `uuidgen` --add a $1:6789 /etc/ceph/monmap +mkdir /var/lib/ceph/mon/ceph-a +ceph-mon -i a --mkfs --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/keyring +cp /var/lib/ceph/mon/keyring /var/lib/ceph/mon/ceph-a +ceph-mon -i a --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring + +# client setup (handy) +cp /var/lib/ceph/mon/keyring /etc/ceph + +# for this test we want to +ceph osd getcrushmap -o /tmp/crushc +crushtool -d /tmp/crushc -o /tmp/crushd +sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /tmp/crushd +crushtool -c /tmp/crushd -o /tmp/crushc +ceph osd setcrushmap -i /tmp/crushc diff --git a/contrib/for-tests/volumes-tester/rbd/osd.sh b/contrib/for-tests/volumes-tester/rbd/osd.sh new file mode 100755 index 00000000000..8daf6492ceb --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/osd.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures and launches a new OSD. +# + +ceph osd create +ceph-osd -i $1 --mkfs --mkkey +ceph auth add osd.$1 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-$1/keyring +ceph osd crush add $1 1 root=default host=cephbox +ceph-osd -i $1 -k /var/lib/ceph/osd/ceph-$1/keyring diff --git a/test/e2e/volumes.go b/test/e2e/volumes.go index 4d39acf1fff..98bc0a47a71 100644 --- a/test/e2e/volumes.go +++ b/test/e2e/volumes.go @@ -400,4 +400,83 @@ var _ = Describe("Volumes", func() { testVolumeClient(c, config, volume, "Hello from iSCSI") }) }) + + //////////////////////////////////////////////////////////////////////// + // Ceph RBD + //////////////////////////////////////////////////////////////////////// + + // Marked with [Skipped] to skip the test by default (see driver.go), + // the test needs privileged containers, which are disabled by default. + // Run the test with "go run hack/e2e.go ... --ginkgo.focus=RBD" + + // Run the test with "go run hack/e2e.go ... --ginkgo.focus=Volume" + Describe("[Skipped] Ceph RBD", func() { + It("should be mountable", func() { + config := VolumeTestConfig{ + namespace: namespace.Name, + prefix: "rbd", + serverImage: "gcr.io/google_containers/volume-rbd", + serverPorts: []int{6789}, + volumes: map[string]string{ + // iSCSI container needs to insert modules from the host + "/lib/modules": "/lib/modules", + "/sys": "/sys", + }, + } + + defer func() { + if clean { + volumeTestCleanup(c, config) + } + }() + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP + Logf("Ceph server IP address: %v", serverIP) + + // create secrets for the server + secret := api.Secret{ + TypeMeta: api.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: api.ObjectMeta{ + Name: config.prefix + "-secret", + }, + Data: map[string][]byte{ + // from contrib/for-tests/volumes-tester/rbd/keyring + "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), + }, + } + + secClient := c.Secrets(config.namespace) + + defer func() { + if clean { + secClient.Delete(config.prefix + "-secret") + } + }() + + if _, err := secClient.Create(&secret); err != nil { + Failf("Failed to create secrets for Ceph RBD: %v", err) + } + + volume := api.VolumeSource{ + RBD: &api.RBDVolumeSource{ + CephMonitors: []string{serverIP}, + RBDPool: "rbd", + RBDImage: "foo", + RadosUser: "admin", + SecretRef: &api.LocalObjectReference{ + Name: config.prefix + "-secret", + }, + FSType: "ext2", + ReadOnly: true, + }, + } + // Must match content of contrib/for-tests/volumes-tester/gluster/index.html + testVolumeClient(c, config, volume, "Hello from RBD") + + }) + }) + })