diff --git a/contrib/for-tests/volumes-tester/iscsi/Dockerfile b/contrib/for-tests/volumes-tester/iscsi/Dockerfile new file mode 100644 index 00000000000..5505faf3aa0 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/Dockerfile @@ -0,0 +1,33 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora +MAINTAINER Jan Safranek, jsafrane@redhat.com +RUN yum install -y iscsi-initiator-utils targetcli net-tools strace && yum clean all +ADD run_iscsid.sh /usr/local/bin/ +ADD initiatorname.iscsi /etc/iscsi/ +ADD block.tar.gz / + +# This JSON file was generated by targetcli with these commands: +# /backstores/fileio create block /block +# /iscsi create +# # Enable demo mode (no authentication!): +# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 +# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block +# saveconfig +ADD saveconfig.json /etc/target/ + +EXPOSE 3260/tcp + +ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"] diff --git a/contrib/for-tests/volumes-tester/iscsi/Makefile b/contrib/for-tests/volumes-tester/iscsi/Makefile new file mode 100644 index 00000000000..93119cdc779 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/Makefile @@ -0,0 +1,22 @@ +all: push + +TAG = 0.1 + +container: + # Build new image and automatically tag it as latest + docker build -t gcr.io/google_containers/volume-iscsi . + # Add the version tag to the latest image + docker tag gcr.io/google_containers/volume-iscsi gcr.io/google_containers/volume-iscsi:$(TAG) + +block: + # Create block.tar.gz with ext2 block device with index.html inside. + # block.tar.gz is already available in git and users don't need to + # regenerate it, this target is here just for reference. + # Run as root! + ./create_block.sh + +push: container + # Push image tagged as latest to repository + gcloud docker push gcr.io/google_containers/volume-iscsi + # Push version tagged image to repository (since this image is already pushed it will simply create or update version tag) + gcloud docker push gcr.io/google_containers/volume-iscsi:$(TAG) diff --git a/contrib/for-tests/volumes-tester/iscsi/README.md b/contrib/for-tests/volumes-tester/iscsi/README.md new file mode 100644 index 00000000000..81233478346 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/README.md @@ -0,0 +1,14 @@ +# iSCSI target container for testing. + +Inspired by https://github.com/rvykydal/dockerfile-iscsid + +* The container needs /lib/modules from the host to insert appropriate + kernel modules for iscsi. This assumes that these modules are installed + on the host! + +* The container needs to run with docker --privileged + +block.tar.gz is a small ext2 filesystem created by `make block` (run as root!) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/for-tests/volumes-tester/iscsi/README.md?pixel)]() diff --git a/contrib/for-tests/volumes-tester/iscsi/block.tar.gz b/contrib/for-tests/volumes-tester/iscsi/block.tar.gz new file mode 100644 index 00000000000..8576d132c8d Binary files /dev/null and b/contrib/for-tests/volumes-tester/iscsi/block.tar.gz differ diff --git a/contrib/for-tests/volumes-tester/iscsi/create_block.sh b/contrib/for-tests/volumes-tester/iscsi/create_block.sh new file mode 100755 index 00000000000..b07e98c9b42 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/create_block.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on the first error. +set -e + +MNTDIR=`mktemp -d` + +cleanup() +{ + # Make sure we return the right exit code + RET=$? + # Silently remove everything and ignore errors + set +e + /bin/umount $MNTDIR 2>/dev/null + /bin/rmdir $MNTDIR 2>/dev/null + /bin/rm block 2>/dev/null + exit $RET +} + +trap cleanup TERM EXIT + +# Create 1MB device with ext2 +dd if=/dev/zero of=block count=1 bs=1M +mkfs.ext2 block + +# Add index.html to it +mount -o loop block $MNTDIR +echo "Hello from iSCSI" > $MNTDIR/index.html +umount $MNTDIR + +rm block.tar.gz 2>/dev/null || : +tar cfz block.tar.gz block diff --git a/contrib/for-tests/volumes-tester/iscsi/initiatorname.iscsi b/contrib/for-tests/volumes-tester/iscsi/initiatorname.iscsi new file mode 100644 index 00000000000..66d3f723c0e --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/initiatorname.iscsi @@ -0,0 +1 @@ +InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5 diff --git a/contrib/for-tests/volumes-tester/iscsi/run_iscsid.sh b/contrib/for-tests/volumes-tester/iscsi/run_iscsid.sh new file mode 100755 index 00000000000..50d38fb5580 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/run_iscsid.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function start() +{ + targetcli restoreconfig + iscsid + echo "iscsid started" +} + +function stop() +{ + echo "Stopping iscsid" + + kill $( cat /var/run/iscsid.pid ) + targetcli clearconfig confirm=True + + echo "iscsid stopped" + exit 0 +} + + +trap stop TERM +start + +while true; do + sleep 5 +done \ No newline at end of file diff --git a/contrib/for-tests/volumes-tester/iscsi/saveconfig.json b/contrib/for-tests/volumes-tester/iscsi/saveconfig.json new file mode 100644 index 00000000000..7520b99e538 --- /dev/null +++ b/contrib/for-tests/volumes-tester/iscsi/saveconfig.json @@ -0,0 +1,102 @@ +{ + "fabric_modules": [], + "storage_objects": [ + { + "attributes": { + "block_size": 512, + "emulate_3pc": 1, + "emulate_caw": 1, + "emulate_dpo": 0, + "emulate_fua_read": 0, + "emulate_fua_write": 1, + "emulate_model_alias": 1, + "emulate_rest_reord": 0, + "emulate_tas": 1, + "emulate_tpu": 0, + "emulate_tpws": 0, + "emulate_ua_intlck_ctrl": 0, + "emulate_write_cache": 1, + "enforce_pr_isids": 1, + "force_pr_aptpl": 0, + "is_nonrot": 0, + "max_unmap_block_desc_count": 1, + "max_unmap_lba_count": 8192, + "max_write_same_len": 4096, + "optimal_sectors": 16384, + "pi_prot_format": 0, + "pi_prot_type": 0, + "queue_depth": 128, + "unmap_granularity": 1, + "unmap_granularity_alignment": 0 + }, + "dev": "block", + "name": "block", + "plugin": "fileio", + "size": 1048576, + "write_back": true, + "wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33" + } + ], + "targets": [ + { + "fabric": "iscsi", + "tpgs": [ + { + "attributes": { + "authentication": 0, + "cache_dynamic_acls": 1, + "default_cmdsn_depth": 64, + "default_erl": 0, + "demo_mode_discovery": 1, + "demo_mode_write_protect": 0, + "generate_node_acls": 1, + "login_timeout": 15, + "netif_timeout": 2, + "prod_mode_write_protect": 0, + "t10_pi": 0 + }, + "enable": true, + "luns": [ + { + "index": 0, + "storage_object": "/backstores/fileio/block" + } + ], + "node_acls": [], + "parameters": { + "AuthMethod": "CHAP,None", + "DataDigest": "CRC32C,None", + "DataPDUInOrder": "Yes", + "DataSequenceInOrder": "Yes", + "DefaultTime2Retain": "20", + "DefaultTime2Wait": "2", + "ErrorRecoveryLevel": "0", + "FirstBurstLength": "65536", + "HeaderDigest": "CRC32C,None", + "IFMarkInt": "2048~65535", + "IFMarker": "No", + "ImmediateData": "Yes", + "InitialR2T": "Yes", + "MaxBurstLength": "262144", + "MaxConnections": "1", + "MaxOutstandingR2T": "1", + "MaxRecvDataSegmentLength": "8192", + "MaxXmitDataSegmentLength": "262144", + "OFMarkInt": "2048~65535", + "OFMarker": "No", + "TargetAlias": "LIO Target" + }, + "portals": [ + { + "ip_address": "0.0.0.0", + "iser": false, + "port": 3260 + } + ], + "tag": 1 + } + ], + "wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c" + } + ] +} diff --git a/contrib/for-tests/volumes-tester/nfs/run_nfs.sh b/contrib/for-tests/volumes-tester/nfs/run_nfs.sh index c22eeb0cdf6..5468c1d7c98 100755 --- a/contrib/for-tests/volumes-tester/nfs/run_nfs.sh +++ b/contrib/for-tests/volumes-tester/nfs/run_nfs.sh @@ -20,9 +20,7 @@ function start() # prepare /etc/exports for i in "$@"; do # fsid=0: needed for NFSv4 - # no_root_squash: root can read/write - # insecure: allow access from ports > 1024 - echo "$i *(rw,fsid=0,no_root_squash,insecure)" >> /etc/exports + echo "$i *(rw,fsid=0,no_root_squash)" >> /etc/exports echo "Serving $i" done diff --git a/contrib/for-tests/volumes-tester/rbd/Dockerfile b/contrib/for-tests/volumes-tester/rbd/Dockerfile new file mode 100644 index 00000000000..2a6498057da --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CEPH all in one +# Based on image by Ricardo Rocha, ricardo@catalyst.net.nz + +FROM fedora +MAINTAINER Jan Safranek jsafrane@redhat.com + +# Base Packages +RUN yum install -y wget ceph ceph-fuse strace && yum clean all + +# Get ports exposed +EXPOSE 6789 + +ADD ./bootstrap.sh /bootstrap.sh +ADD ./mon.sh /mon.sh +ADD ./osd.sh /osd.sh +ADD ./ceph.conf.sh /ceph.conf.sh +ADD ./keyring /var/lib/ceph/mon/keyring +ADD ./block.tar.gz / + +CMD /bootstrap.sh diff --git a/contrib/for-tests/volumes-tester/rbd/Makefile b/contrib/for-tests/volumes-tester/rbd/Makefile new file mode 100644 index 00000000000..a1d1e168828 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/Makefile @@ -0,0 +1,22 @@ +all: push + +TAG = 0.1 + +container: + # Build new image and automatically tag it as latest + docker build -t gcr.io/google_containers/volume-rbd . + # Add the version tag to the latest image + docker tag gcr.io/google_containers/volume-rbd gcr.io/google_containers/volume-rbd:$(TAG) + +block: + # Create block.tar.gz with ext2 block device with index.html inside. + # block.tar.gz is already available in git and users don't need to + # regenerate it, this target is here just for reference. + # Run as root! + ./create_block.sh + +push: container + # Push image tagged as latest to repository + gcloud docker push gcr.io/google_containers/volume-rbd + # Push version tagged image to repository (since this image is already pushed it will simply create or update version tag) + gcloud docker push gcr.io/google_containers/volume-rbd:$(TAG) diff --git a/contrib/for-tests/volumes-tester/rbd/block.tar.gz b/contrib/for-tests/volumes-tester/rbd/block.tar.gz new file mode 100644 index 00000000000..395d1492f9b Binary files /dev/null and b/contrib/for-tests/volumes-tester/rbd/block.tar.gz differ diff --git a/contrib/for-tests/volumes-tester/rbd/bootstrap.sh b/contrib/for-tests/volumes-tester/rbd/bootstrap.sh new file mode 100755 index 00000000000..f5ca9dd18d1 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/bootstrap.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Bootstraps a CEPH server. +# It creates two OSDs on local machine, creates RBD pool there +# and imports 'block' device there. +# +# We must create fresh OSDs and filesystem here, because shipping it +# in a container would increase the image by ~300MB. +# + + +# Create /etc/ceph/ceph.conf +sh ./ceph.conf.sh `hostname -i` + +# Configure and start ceph-mon +sh ./mon.sh `hostname -i` + +# Configure and start 2x ceph-osd +mkdir -p /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-1 +sh ./osd.sh 0 +sh ./osd.sh 1 + +# Prepare a RBD volume +# NOTE: we need Ceph kernel modules on the host! +rbd import block foo + +echo "Ceph is ready" + +# Wait forever +while true; do + sleep 10 +done \ No newline at end of file diff --git a/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh b/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh new file mode 100755 index 00000000000..f3f557585c5 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/ceph.conf.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures /etc/ceph.conf from a template. +# + +echo " +[global] +auth cluster required = none +auth service required = none +auth client required = none + +[mon.a] +host = cephbox +mon addr = $1 + +[osd] +osd journal size = 128 +journal dio = false + +[osd.0] +osd host = cephbox +" > /etc/ceph/ceph.conf + diff --git a/contrib/for-tests/volumes-tester/rbd/create_block.sh b/contrib/for-tests/volumes-tester/rbd/create_block.sh new file mode 100755 index 00000000000..87c7eeaba74 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/create_block.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create block.tar.gz with a small ext2 filesystem. +# It must be run as root (to mount with '-o loop')! + +# Exit on the first error. +set -e + +MNTDIR=`mktemp -d` + +cleanup() +{ + # Make sure we return the right exit code + RET=$? + # Silently remove everything and ignore errors + set +e + /bin/umount $MNTDIR 2>/dev/null + /bin/rmdir $MNTDIR 2>/dev/null + /bin/rm block 2>/dev/null + exit $RET +} + +trap cleanup TERM EXIT + +# Create 1MB device with ext2 +dd if=/dev/zero of=block count=1 bs=1M +mkfs.ext2 block + +# Add index.html to it +mount -o loop block $MNTDIR +echo "Hello from RBD" > $MNTDIR/index.html +umount $MNTDIR + +rm block.tar.gz 2>/dev/null || : +tar cfz block.tar.gz block diff --git a/contrib/for-tests/volumes-tester/rbd/keyring b/contrib/for-tests/volumes-tester/rbd/keyring new file mode 100644 index 00000000000..f8c5af22417 --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/keyring @@ -0,0 +1,8 @@ +[mon.] + key = AQDRrKNV6z4UChAABzP1ZyysTU4pjgjNOf/p3A== +[client.admin] + key = AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA== + auid = 0 + caps mds = "allow *" + caps mon = "allow *" + caps osd = "allow *" diff --git a/contrib/for-tests/volumes-tester/rbd/mon.sh b/contrib/for-tests/volumes-tester/rbd/mon.sh new file mode 100755 index 00000000000..8013044b89f --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/mon.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures and launches a new MON. +# + +# monitor setup +monmaptool --create --clobber --fsid `uuidgen` --add a $1:6789 /etc/ceph/monmap +mkdir /var/lib/ceph/mon/ceph-a +ceph-mon -i a --mkfs --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/keyring +cp /var/lib/ceph/mon/keyring /var/lib/ceph/mon/ceph-a +ceph-mon -i a --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring + +# client setup (handy) +cp /var/lib/ceph/mon/keyring /etc/ceph + +# for this test we want to +ceph osd getcrushmap -o /tmp/crushc +crushtool -d /tmp/crushc -o /tmp/crushd +sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /tmp/crushd +crushtool -c /tmp/crushd -o /tmp/crushc +ceph osd setcrushmap -i /tmp/crushc diff --git a/contrib/for-tests/volumes-tester/rbd/osd.sh b/contrib/for-tests/volumes-tester/rbd/osd.sh new file mode 100755 index 00000000000..8daf6492ceb --- /dev/null +++ b/contrib/for-tests/volumes-tester/rbd/osd.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures and launches a new OSD. +# + +ceph osd create +ceph-osd -i $1 --mkfs --mkkey +ceph auth add osd.$1 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-$1/keyring +ceph osd crush add $1 1 root=default host=cephbox +ceph-osd -i $1 -k /var/lib/ceph/osd/ceph-$1/keyring diff --git a/test/e2e/persistent_volumes.go b/test/e2e/persistent_volumes.go index 4514669f458..6b19705c437 100644 --- a/test/e2e/persistent_volumes.go +++ b/test/e2e/persistent_volumes.go @@ -63,7 +63,8 @@ var _ = Describe("[Skipped] persistentVolumes", func() { volumeTestCleanup(c, config) }() - serverIP := startVolumeServer(c, config) + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP Logf("NFS server IP address: %v", serverIP) pv := makePersistentVolume(serverIP) diff --git a/test/e2e/volumes.go b/test/e2e/volumes.go index 1f75877a2e0..98bc0a47a71 100644 --- a/test/e2e/volumes.go +++ b/test/e2e/volumes.go @@ -35,7 +35,6 @@ import ( "fmt" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client" - "k8s.io/kubernetes/pkg/util" "time" . "github.com/onsi/ginkgo" @@ -53,18 +52,19 @@ type VolumeTestConfig struct { serverImage string // Ports to export from the server pod. TCP only. serverPorts []int + // Volumes needed to be mounted to the server container from the host + // map -> + volumes map[string]string } // Starts a container specified by config.serverImage and exports all -// config.serverPorts from it via a service. The function returns IP -// address of the service. -func startVolumeServer(client *client.Client, config VolumeTestConfig) string { +// config.serverPorts from it. The returned pod should be used to get the server +// IP address and create appropriate VolumeSource. +func startVolumeServer(client *client.Client, config VolumeTestConfig) *api.Pod { podClient := client.Pods(config.namespace) - serviceClient := client.Services(config.namespace) portCount := len(config.serverPorts) serverPodPorts := make([]api.ContainerPort, portCount) - servicePorts := make([]api.ServicePort, portCount) for i := 0; i < portCount; i++ { portName := fmt.Sprintf("%s-%d", config.prefix, i) @@ -74,12 +74,25 @@ func startVolumeServer(client *client.Client, config VolumeTestConfig) string { ContainerPort: config.serverPorts[i], Protocol: api.ProtocolTCP, } - servicePorts[i] = api.ServicePort{ - Name: portName, - Protocol: "TCP", - Port: config.serverPorts[i], - TargetPort: util.NewIntOrStringFromInt(config.serverPorts[i]), + } + + volumeCount := len(config.volumes) + volumes := make([]api.Volume, volumeCount) + mounts := make([]api.VolumeMount, volumeCount) + + i := 0 + for src, dst := range config.volumes { + mountName := fmt.Sprintf("path%d", i) + volumes[i].Name = mountName + volumes[i].VolumeSource.HostPath = &api.HostPathVolumeSource{ + Path: src, } + + mounts[i].Name = mountName + mounts[i].ReadOnly = false + mounts[i].MountPath = dst + + i++ } By(fmt.Sprint("creating ", config.prefix, " server pod")) @@ -105,9 +118,11 @@ func startVolumeServer(client *client.Client, config VolumeTestConfig) string { SecurityContext: &api.SecurityContext{ Privileged: privileged, }, - Ports: serverPodPorts, + Ports: serverPodPorts, + VolumeMounts: mounts, }, }, + Volumes: volumes, }, } _, err := podClient.Create(serverPod) @@ -115,26 +130,13 @@ func startVolumeServer(client *client.Client, config VolumeTestConfig) string { expectNoError(waitForPodRunningInNamespace(client, serverPod.Name, config.namespace)) - By(fmt.Sprint("creating ", config.prefix, " service")) - service := &api.Service{ - ObjectMeta: api.ObjectMeta{ - Name: config.prefix + "-server", - }, - Spec: api.ServiceSpec{ - Selector: map[string]string{ - "role": config.prefix + "-server", - }, - Ports: servicePorts, - }, - } - createdService, err := serviceClient.Create(service) - expectNoError(err, "Failed to create %s service: %v", service.Name, err) + By("locating the server pod") + pod, err := podClient.Get(serverPod.Name) + expectNoError(err, "Cannot locate the server pod %v: %v", serverPod.Name, err) By("sleeping a bit to give the server time to start") time.Sleep(20 * time.Second) - - ip := createdService.Spec.ClusterIP - return ip + return pod } // Clean both server and client pods. @@ -144,11 +146,9 @@ func volumeTestCleanup(client *client.Client, config VolumeTestConfig) { defer GinkgoRecover() podClient := client.Pods(config.namespace) - serviceClient := client.Services(config.namespace) // ignore all errors, the pods may not be even created podClient.Delete(config.prefix+"-client", nil) - serviceClient.Delete(config.prefix + "-server") podClient.Delete(config.prefix+"-server", nil) } @@ -261,7 +261,8 @@ var _ = Describe("Volumes", func() { volumeTestCleanup(c, config) } }() - serverIP := startVolumeServer(c, config) + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP Logf("NFS server IP address: %v", serverIP) volume := api.VolumeSource{ @@ -297,9 +298,49 @@ var _ = Describe("Volumes", func() { volumeTestCleanup(c, config) } }() - serverIP := startVolumeServer(c, config) + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP Logf("Gluster server IP address: %v", serverIP) + // create Endpoints for the server + endpoints := api.Endpoints{ + TypeMeta: api.TypeMeta{ + Kind: "Endpoints", + APIVersion: "v1", + }, + ObjectMeta: api.ObjectMeta{ + Name: config.prefix + "-server", + }, + Subsets: []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{ + { + IP: serverIP, + }, + }, + Ports: []api.EndpointPort{ + { + Name: "gluster", + Port: 24007, + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + } + + endClient := c.Endpoints(config.namespace) + + defer func() { + if clean { + endClient.Delete(config.prefix + "-server") + } + }() + + if _, err := endClient.Create(&endpoints); err != nil { + Failf("Failed to create endpoints for Gluster server: %v", err) + } + volume := api.VolumeSource{ Glusterfs: &api.GlusterfsVolumeSource{ EndpointsName: config.prefix + "-server", @@ -312,4 +353,130 @@ var _ = Describe("Volumes", func() { testVolumeClient(c, config, volume, "Hello from GlusterFS!") }) }) + + //////////////////////////////////////////////////////////////////////// + // iSCSI + //////////////////////////////////////////////////////////////////////// + + // Marked with [Skipped] to skip the test by default (see driver.go), + // the test needs privileged containers, which are disabled by default. + // Also, make sure that iscsiadm utility and iscsi target kernel modules + // are installed on all nodes! + // Run the test with "go run hack/e2e.go ... --ginkgo.focus=iSCSI" + + Describe("[Skipped] iSCSI", func() { + It("should be mountable", func() { + config := VolumeTestConfig{ + namespace: namespace.Name, + prefix: "iscsi", + serverImage: "gcr.io/google_containers/volume-iscsi", + serverPorts: []int{3260}, + volumes: map[string]string{ + // iSCSI container needs to insert modules from the host + "/lib/modules": "/lib/modules", + }, + } + + defer func() { + if clean { + volumeTestCleanup(c, config) + } + }() + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP + Logf("iSCSI server IP address: %v", serverIP) + + volume := api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: serverIP + ":3260", + // from contrib/for-tests/volumes-tester/iscsi/initiatorname.iscsi + IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", + Lun: 0, + FSType: "ext2", + ReadOnly: true, + }, + } + // Must match content of contrib/for-tests/volumes-tester/iscsi/block.tar.gz + testVolumeClient(c, config, volume, "Hello from iSCSI") + }) + }) + + //////////////////////////////////////////////////////////////////////// + // Ceph RBD + //////////////////////////////////////////////////////////////////////// + + // Marked with [Skipped] to skip the test by default (see driver.go), + // the test needs privileged containers, which are disabled by default. + // Run the test with "go run hack/e2e.go ... --ginkgo.focus=RBD" + + // Run the test with "go run hack/e2e.go ... --ginkgo.focus=Volume" + Describe("[Skipped] Ceph RBD", func() { + It("should be mountable", func() { + config := VolumeTestConfig{ + namespace: namespace.Name, + prefix: "rbd", + serverImage: "gcr.io/google_containers/volume-rbd", + serverPorts: []int{6789}, + volumes: map[string]string{ + // iSCSI container needs to insert modules from the host + "/lib/modules": "/lib/modules", + "/sys": "/sys", + }, + } + + defer func() { + if clean { + volumeTestCleanup(c, config) + } + }() + pod := startVolumeServer(c, config) + serverIP := pod.Status.PodIP + Logf("Ceph server IP address: %v", serverIP) + + // create secrets for the server + secret := api.Secret{ + TypeMeta: api.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: api.ObjectMeta{ + Name: config.prefix + "-secret", + }, + Data: map[string][]byte{ + // from contrib/for-tests/volumes-tester/rbd/keyring + "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), + }, + } + + secClient := c.Secrets(config.namespace) + + defer func() { + if clean { + secClient.Delete(config.prefix + "-secret") + } + }() + + if _, err := secClient.Create(&secret); err != nil { + Failf("Failed to create secrets for Ceph RBD: %v", err) + } + + volume := api.VolumeSource{ + RBD: &api.RBDVolumeSource{ + CephMonitors: []string{serverIP}, + RBDPool: "rbd", + RBDImage: "foo", + RadosUser: "admin", + SecretRef: &api.LocalObjectReference{ + Name: config.prefix + "-secret", + }, + FSType: "ext2", + ReadOnly: true, + }, + } + // Must match content of contrib/for-tests/volumes-tester/gluster/index.html + testVolumeClient(c, config, volume, "Hello from RBD") + + }) + }) + })