Add basic install and mount flexvolumes e2e tests

This commit is contained in:
Matthew Wong 2017-06-30 17:37:38 -04:00
parent dd819b5013
commit 55a417e84f
7 changed files with 504 additions and 6 deletions

View File

@ -152,6 +152,16 @@ CONTROLLER_MANAGER_TEST_LOG_LEVEL="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CL
SCHEDULER_TEST_LOG_LEVEL="${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
# TODO: change this and flex e2e test when default flex volume install path is changed for GCI
# Set flex dir to one that's readable from controller-manager container and writable by the flex e2e test.
if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
CONTROLLER_MANAGER_TEST_VOLUME_PLUGIN_DIR="--flex-volume-plugin-dir=/etc/srv/kubernetes/kubelet-plugins/volume/exec"
fi
# Set flex dir to one that's readable from kubelet and writable by the flex e2e test.
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || ([[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]] && [[ "${REGISTER_MASTER_KUBELET}" == "false" ]]); then
KUBELET_TEST_VOLUME_PLUGIN_DIR="--volume-plugin-dir=/etc/srv/kubernetes/kubelet-plugins/volume/exec"
fi
TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=1}"
TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}"
TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m}"
@ -159,7 +169,7 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m
# ContentType used by all components to communicate with apiserver.
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBELET_TEST_VOLUME_PLUGIN_DIR:-}"
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
fi
@ -167,7 +177,7 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${MASTER_OS_DISTRIBUTION}"
MASTER_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
fi
APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}"
CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}"
CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${CONTROLLER_MANAGER_TEST_VOLUME_PLUGIN_DIR:-}"
SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"

View File

@ -342,6 +342,12 @@ func SkipUnlessProviderIs(supportedProviders ...string) {
}
}
func SkipUnlessMasterOSDistroIs(supportedMasterOsDistros ...string) {
if !MasterOSDistroIs(supportedMasterOsDistros...) {
Skipf("Only supported for master OS distro %v (not %s)", supportedMasterOsDistros, TestContext.MasterOSDistro)
}
}
func SkipUnlessNodeOSDistroIs(supportedNodeOsDistros ...string) {
if !NodeOSDistroIs(supportedNodeOsDistros...) {
Skipf("Only supported for node OS distro %v (not %s)", supportedNodeOsDistros, TestContext.NodeOSDistro)
@ -383,6 +389,15 @@ func ProviderIs(providers ...string) bool {
return false
}
func MasterOSDistroIs(supportedMasterOsDistros ...string) bool {
for _, distro := range supportedMasterOsDistros {
if strings.ToLower(distro) == strings.ToLower(TestContext.MasterOSDistro) {
return true
}
}
return false
}
func NodeOSDistroIs(supportedNodeOsDistros ...string) bool {
for _, distro := range supportedNodeOsDistros {
if strings.ToLower(distro) == strings.ToLower(TestContext.NodeOSDistro) {
@ -3758,6 +3773,43 @@ func RestartKubeProxy(host string) error {
return nil
}
func RestartKubelet(host string) error {
// TODO: Make it work for all providers and distros.
if !ProviderIs("gce", "aws") {
return fmt.Errorf("unsupported provider: %s", TestContext.Provider)
}
if ProviderIs("gce") && !NodeOSDistroIs("debian", "gci") {
return fmt.Errorf("unsupported node OS distro: %s", TestContext.NodeOSDistro)
}
var cmd string
if ProviderIs("gce") && NodeOSDistroIs("debian") {
cmd = "sudo /etc/init.d/kubelet restart"
} else {
cmd = "sudo systemctl restart kubelet"
}
Logf("Restarting kubelet via ssh, running: %v", cmd)
result, err := SSH(cmd, host, TestContext.Provider)
if err != nil || result.Code != 0 {
LogSSHResult(result)
return fmt.Errorf("couldn't restart kubelet: %v", err)
}
return nil
}
func WaitForKubeletUp(host string) error {
cmd := "curl http://localhost:" + strconv.Itoa(ports.KubeletReadOnlyPort) + "/healthz"
for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
result, err := SSH(cmd, host, TestContext.Provider)
if err != nil || result.Code != 0 {
LogSSHResult(result)
}
if result.Stdout == "ok" {
return nil
}
}
return fmt.Errorf("waiting for kubelet timed out")
}
func RestartApiserver(c discovery.ServerVersionInterface) error {
// TODO: Make it work for all providers.
if !ProviderIs("gce", "gke", "aws") {
@ -3806,6 +3858,38 @@ func WaitForApiserverUp(c clientset.Interface) error {
return fmt.Errorf("waiting for apiserver timed out")
}
func RestartControllerManager() error {
// TODO: Make it work for all providers and distros.
if !ProviderIs("gce", "aws") {
return fmt.Errorf("unsupported provider: %s", TestContext.Provider)
}
if ProviderIs("gce") && !MasterOSDistroIs("gci") {
return fmt.Errorf("unsupported master OS distro: %s", TestContext.MasterOSDistro)
}
cmd := "sudo docker ps | grep k8s_kube-controller-manager | cut -d ' ' -f 1 | xargs sudo docker kill"
Logf("Restarting controller-manager via ssh, running: %v", cmd)
result, err := SSH(cmd, GetMasterHost()+":22", TestContext.Provider)
if err != nil || result.Code != 0 {
LogSSHResult(result)
return fmt.Errorf("couldn't restart controller-manager: %v", err)
}
return nil
}
func WaitForControllerManagerUp() error {
cmd := "curl http://localhost:" + strconv.Itoa(ports.ControllerManagerPort) + "/healthz"
for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
result, err := SSH(cmd, GetMasterHost()+":22", TestContext.Provider)
if err != nil || result.Code != 0 {
LogSSHResult(result)
}
if result.Stdout == "ok" {
return nil
}
}
return fmt.Errorf("waiting for controller-manager timed out")
}
// WaitForClusterSize waits until the cluster has desired size and there is no not-ready nodes in it.
// By cluster size we mean number of Nodes excluding Master Node.
func WaitForClusterSize(c clientset.Interface, size int, timeout time.Duration) error {

View File

@ -96,8 +96,10 @@ type VolumeTestConfig struct {
// Wait for the pod to terminate successfully
// False indicates that the pod is long running
WaitForCompletion bool
// NodeName to run pod on. Default is any node.
NodeName string
// ServerNodeName is the spec.nodeName to run server pod on. Default is any node.
ServerNodeName string
// ClientNodeName is the spec.nodeName to run client pod on. Default is any node.
ClientNodeName string
}
// VolumeTest contains a volume to mount into a client pod and its
@ -283,7 +285,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.
},
Volumes: volumes,
RestartPolicy: restartPolicy,
NodeName: config.NodeName,
NodeName: config.ServerNodeName,
},
}
@ -389,6 +391,7 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
},
},
Volumes: []v1.Volume{},
NodeName: config.ClientNodeName,
},
}
podsNamespacer := client.CoreV1().Pods(config.Namespace)

View File

@ -11,6 +11,7 @@ go_library(
name = "go_default_library",
srcs = [
"empty_dir_wrapper.go",
"flexvolume.go",
"framework.go",
"pd.go",
"persistent_volumes.go",
@ -39,6 +40,7 @@ go_library(
"//pkg/kubelet/apis:go_default_library",
"//pkg/volume/util/volumehelper:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/generated:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",

View File

@ -0,0 +1,207 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package storage
import (
"fmt"
"math/rand"
"net"
"path"
. "github.com/onsi/ginkgo"
"k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/generated"
)
const (
sshPort = "22"
driverDir = "test/e2e/testing-manifests/flexvolume/"
defaultVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
// TODO: change this and config-test.sh when default flex volume install path is changed for GCI
// On gci, root is read-only and controller-manager containerized. Assume
// controller-manager has started with --flex-volume-plugin-dir equal to this
// (see cluster/gce/config-test.sh)
gciVolumePluginDir = "/etc/srv/kubernetes/kubelet-plugins/volume/exec"
)
// testFlexVolume tests that a client pod using a given flexvolume driver
// successfully mounts it and runs
func testFlexVolume(driver string, cs clientset.Interface, config framework.VolumeTestConfig, f *framework.Framework, clean bool) {
tests := []framework.VolumeTest{
{
Volume: v1.VolumeSource{
FlexVolume: &v1.FlexVolumeSource{
Driver: "k8s/" + driver,
},
},
File: "index.html",
// Must match content of examples/volumes/flexvolume/dummy(-attachable) domount
ExpectedContent: "Hello from flexvolume!",
},
}
framework.TestVolumeClient(cs, config, nil, tests)
if clean {
framework.VolumeTestCleanup(f, config)
}
}
// installFlex installs the driver found at filePath on the node and restarts
// kubelet. If node is nil, installs on the master and restarts
// controller-manager.
func installFlex(node *v1.Node, vendor, driver, filePath string) {
flexDir := getFlexDir(node == nil, vendor, driver)
flexFile := path.Join(flexDir, driver)
host := ""
if node != nil {
host = framework.GetNodeExternalIP(node)
} else {
host = net.JoinHostPort(framework.GetMasterHost(), sshPort)
}
cmd := fmt.Sprintf("sudo mkdir -p %s", flexDir)
sshAndLog(cmd, host)
data := generated.ReadOrDie(filePath)
cmd = fmt.Sprintf("sudo tee <<'EOF' %s\n%s\nEOF", flexFile, string(data))
sshAndLog(cmd, host)
cmd = fmt.Sprintf("sudo chmod +x %s", flexFile)
sshAndLog(cmd, host)
if node != nil {
err := framework.RestartKubelet(host)
framework.ExpectNoError(err)
err = framework.WaitForKubeletUp(host)
framework.ExpectNoError(err)
} else {
err := framework.RestartControllerManager()
framework.ExpectNoError(err)
err = framework.WaitForControllerManagerUp()
framework.ExpectNoError(err)
}
}
func uninstallFlex(node *v1.Node, vendor, driver string) {
flexDir := getFlexDir(node == nil, vendor, driver)
host := ""
if node != nil {
host = framework.GetNodeExternalIP(node)
} else {
host = net.JoinHostPort(framework.GetMasterHost(), sshPort)
}
cmd := fmt.Sprintf("sudo rm -r %s", flexDir)
sshAndLog(cmd, host)
}
func getFlexDir(master bool, vendor, driver string) string {
volumePluginDir := defaultVolumePluginDir
if framework.ProviderIs("gce") {
if (master && framework.MasterOSDistroIs("gci")) || (!master && framework.NodeOSDistroIs("gci")) {
volumePluginDir = gciVolumePluginDir
}
}
flexDir := path.Join(volumePluginDir, fmt.Sprintf("/%s~%s/", vendor, driver))
return flexDir
}
func sshAndLog(cmd, host string) {
result, err := framework.SSH(cmd, host, framework.TestContext.Provider)
framework.LogSSHResult(result)
framework.ExpectNoError(err)
if result.Code != 0 {
framework.Failf("%s returned non-zero, stderr: %s", cmd, result.Stderr)
}
}
var _ = SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() {
f := framework.NewDefaultFramework("flexvolume")
// If 'false', the test won't clear its volumes upon completion. Useful for debugging,
// note that namespace deletion is handled by delete-namespace flag
clean := true
var cs clientset.Interface
var ns *v1.Namespace
var node v1.Node
var config framework.VolumeTestConfig
var suffix string
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce")
framework.SkipUnlessMasterOSDistroIs("gci")
framework.SkipUnlessNodeOSDistroIs("debian", "gci")
framework.SkipUnlessSSHKeyPresent()
cs = f.ClientSet
ns = f.Namespace
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
node = nodes.Items[rand.Intn(len(nodes.Items))]
config = framework.VolumeTestConfig{
Namespace: ns.Name,
Prefix: "flex",
ClientNodeName: node.Name,
}
suffix = ns.Name
})
It("should be mountable when non-attachable", func() {
driver := "dummy"
driverInstallAs := driver + "-" + suffix
By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
installFlex(&node, "k8s", driverInstallAs, path.Join(driverDir, driver))
testFlexVolume(driverInstallAs, cs, config, f, clean)
By("waiting for flex client pod to terminate")
if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
framework.ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
}
By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
uninstallFlex(&node, "k8s", driverInstallAs)
})
It("should be mountable when attachable", func() {
driver := "dummy-attachable"
driverInstallAs := driver + "-" + suffix
By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
installFlex(&node, "k8s", driverInstallAs, path.Join(driverDir, driver))
By(fmt.Sprintf("installing flexvolume %s on master as %s", path.Join(driverDir, driver), driverInstallAs))
installFlex(nil, "k8s", driverInstallAs, path.Join(driverDir, driver))
testFlexVolume(driverInstallAs, cs, config, f, clean)
By("waiting for flex client pod to terminate")
if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
framework.ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
}
By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
uninstallFlex(&node, "k8s", driverInstallAs)
By(fmt.Sprintf("uninstalling flexvolume %s from master", driverInstallAs))
uninstallFlex(nil, "k8s", driverInstallAs)
})
})

View File

@ -0,0 +1,68 @@
#!/bin/sh
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FLEX_DUMMY_LOG=${FLEX_DUMMY_LOG:-"/tmp/flex-dummy.log"}
log() {
printf "$*" >&1
}
debug() {
echo "$(date) $*" >> "${FLEX_DUMMY_LOG}"
}
domount() {
debug "domount $@"
MNTPATH=$1
mkdir -p ${MNTPATH} >/dev/null 2>&1
mount -t tmpfs none ${MNTPATH} >/dev/null 2>&1
echo "Hello from flexvolume!" >> "${MNTPATH}/index.html"
log "{\"status\":\"Success\"}"
exit 0
}
unmount() {
debug "unmount $@"
MNTPATH=$1
rm ${MNTPATH}/index.html >/dev/null 2>&1
umount ${MNTPATH} >/dev/null 2>&1
log "{\"status\":\"Success\"}"
exit 0
}
op=$1
if [ "$op" = "init" ]; then
debug "init $@"
log "{\"status\":\"Success\",\"capabilities\":{\"attach\":false}}"
exit 0
fi
shift
case "$op" in
mount)
domount $*
;;
unmount)
unmount $*
;;
*)
log "{\"status\":\"Not supported\"}"
exit 0
esac
exit 1

View File

@ -0,0 +1,124 @@
#!/bin/sh
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FLEX_DUMMY_LOG=${FLEX_DUMMY_LOG:-"/tmp/flex-dummy.log"}
VALID_MNTDEVICE=foo
# attach always returns one valid mount device so a different device
# showing up in a subsequent driver call implies a bug
validateMountDeviceOrDie() {
MNTDEVICE=$1
CALL=$2
if [ "$MNTDEVICE" != "$VALID_MNTDEVICE" ]; then
log "{\"status\":\"Failure\",\"message\":\"call "${CALL}" expected device "${VALID_MNTDEVICE}", got device "${MNTDEVICE}"\"}"
exit 0
fi
}
log() {
printf "$*" >&1
}
debug() {
echo "$(date) $*" >> "${FLEX_DUMMY_LOG}"
}
attach() {
debug "attach $@"
log "{\"status\":\"Success\",\"device\":\""${VALID_MNTDEVICE}"\"}"
exit 0
}
detach() {
debug "detach $@"
# TODO issue 44737 detach is passed PV name, not mount device
log "{\"status\":\"Success\"}"
exit 0
}
waitforattach() {
debug "waitforattach $@"
MNTDEVICE=$1
validateMountDeviceOrDie "$MNTDEVICE" "waitforattach"
log "{\"status\":\"Success\",\"device\":\""${MNTDEVICE}"\"}"
exit 0
}
isattached() {
debug "isattached $@"
log "{\"status\":\"Success\",\"attached\":true}"
exit 0
}
domountdevice() {
debug "domountdevice $@"
MNTDEVICE=$2
validateMountDeviceOrDie "$MNTDEVICE" "domountdevice"
MNTPATH=$1
mkdir -p ${MNTPATH} >/dev/null 2>&1
mount -t tmpfs none ${MNTPATH} >/dev/null 2>&1
echo "Hello from flexvolume!" >> "${MNTPATH}/index.html"
log "{\"status\":\"Success\"}"
exit 0
}
unmountdevice() {
debug "unmountdevice $@"
MNTDEVICE=$2
validateMountDeviceOrDie "$MNTDEVICE" "unmountdevice"
MNTPATH=$1
rm "${MNTPATH}/index.html" >/dev/null 2>&1
umount ${MNTPATH} >/dev/null 2>&1
log "{\"status\":\"Success\"}"
exit 0
}
op=$1
if [ "$op" = "init" ]; then
debug "init $@"
log "{\"status\":\"Success\",\"capabilities\":{\"attach\":true}}"
exit 0
fi
shift
case "$op" in
attach)
attach $*
;;
detach)
detach $*
;;
waitforattach)
waitforattach $*
;;
isattached)
isattached $*
;;
mountdevice)
domountdevice $*
;;
unmountdevice)
unmountdevice $*
;;
*)
log "{\"status\":\"Not supported\"}"
exit 0
esac
exit 1