Merge pull request #72836 from pohly/storage-test-plugin

e2e/storage: testing of external storage drivers

test/e2e/BUILD

@@ -33,6 +33,7 @@ go_test(
         "//test/e2e/scheduling:go_default_library",
         "//test/e2e/servicecatalog:go_default_library",
         "//test/e2e/storage:go_default_library",
+        "//test/e2e/storage/external:go_default_library",
         "//test/e2e/ui:go_default_library",
         "//test/e2e/windows:go_default_library",
         "//test/utils/image:go_default_library",

test/e2e/e2e_test.go

@@ -51,6 +51,7 @@ import (
 	_ "k8s.io/kubernetes/test/e2e/scheduling"
 	_ "k8s.io/kubernetes/test/e2e/servicecatalog"
 	_ "k8s.io/kubernetes/test/e2e/storage"
+	_ "k8s.io/kubernetes/test/e2e/storage/external"
 	_ "k8s.io/kubernetes/test/e2e/ui"
 	_ "k8s.io/kubernetes/test/e2e/windows"
 )

test/e2e/storage/BUILD

@@ -95,6 +95,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//test/e2e/storage/drivers:all-srcs",
+        "//test/e2e/storage/external:all-srcs",
         "//test/e2e/storage/testpatterns:all-srcs",
         "//test/e2e/storage/testsuites:all-srcs",
         "//test/e2e/storage/utils:all-srcs",

test/e2e/storage/external/BUILD

@@ -0,0 +1,48 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["external.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/storage/external",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//staging/src/k8s.io/api/storage/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/storage/testpatterns:go_default_library",
+        "//test/e2e/storage/testsuites:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["external_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//test/e2e/storage/testsuites:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/storage/external/README.md

@@ -0,0 +1,49 @@
+When a test suite like test/e2e/e2e.test from Kubernetes includes this
+package, the -storage.testdriver parameter can be used one or more
+times to enabling testing of a certain pre-installed storage driver.
+
+The parameter takes as argument the name of a .yaml or .json file. The
+filename can be absolute or relative to --repo-root. The content of
+the file is used to populate a struct that defines how to test the
+driver. For a full definition of the struct see the external.go file.
+
+Here is an example for the CSI hostpath driver:
+
+    ShortName: mytest
+    StorageClass:
+      FromName: true
+    SnapshotClass:
+      FromName: true
+    DriverInfo:
+      Name: csi-hostpath
+      Capabilities:
+        persistence: true
+        dataSource: true
+        multipods: true
+
+Currently there is no checking for unknown fields, i.e. only file
+entries that match with struct entries are used and other entries are
+silently ignored, so beware of typos.
+
+For each driver, the storage tests from `test/e2e/storage/testsuites`
+are added for that driver with `External Storage [Driver: <Name>]` as
+prefix.
+
+To run just those tests for the example above, put that content into
+`/tmp/hostpath-testdriver.yaml` and invoke:
+
+    ginkgo -p -focus='External.Storage.*csi-hostpath' \
+           -skip='\[Feature:|\[Disruptive\]' \
+           ./test/e2e \
+           -- \
+           -storage.testdriver=/tmp/hostpath-testdriver.yaml
+
+This disables tests which depend on optional features. Those tests
+must be run by selecting them explicitly in an environment that
+supports them, for example snapshotting:
+
+    ginkgo -p -focus='External.Storage.*csi-hostpath.*\[Feature:VolumeSnapshotDataSource\]' \
+           -skip='\[Disruptive\]' \
+           ./test/e2e \
+           -- \
+           -storage.testdriver=/tmp/hostpath-testdriver.yaml

test/e2e/storage/external/external.go

@@ -0,0 +1,276 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package external
+
+import (
+	"flag"
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+// List of testSuites to be executed for each external driver.
+var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitProvisioningTestSuite,
+	testsuites.InitSnapshottableTestSuite,
+	testsuites.InitSubPathTestSuite,
+	testsuites.InitVolumeIOTestSuite,
+	testsuites.InitVolumeModeTestSuite,
+	testsuites.InitVolumesTestSuite,
+}
+
+func init() {
+	flag.Var(testDriverParameter{}, "storage.testdriver", "name of a .yaml or .json file that defines a driver for storage testing, can be used more than once")
+}
+
+// testDriverParameter is used to hook loading of the driver
+// definition file and test instantiation into argument parsing: for
+// each of potentially many parameters, Set is called and then does
+// both immediately. There is no other code location between argument
+// parsing and starting of the test suite where those test could be
+// defined.
+type testDriverParameter struct {
+}
+
+var _ flag.Value = testDriverParameter{}
+
+func (t testDriverParameter) String() string {
+	return "<.yaml or .json file>"
+}
+
+func (t testDriverParameter) Set(filename string) error {
+	driver, err := t.loadDriverDefinition(filename)
+	if err != nil {
+		return err
+	}
+	if driver.DriverInfo.Name == "" {
+		return errors.Errorf("%q: DriverInfo.Name not set", filename)
+	}
+
+	description := "External Storage " + testsuites.GetDriverNameWithFeatureTags(driver)
+	ginkgo.Describe(description, func() {
+		testsuites.DefineTestSuite(driver, csiTestSuites)
+	})
+
+	return nil
+}
+
+func (t testDriverParameter) loadDriverDefinition(filename string) (*driverDefinition, error) {
+	if filename == "" {
+		return nil, errors.New("missing file name")
+	}
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	// Some reasonable defaults follow.
+	driver := &driverDefinition{
+		DriverInfo: testsuites.DriverInfo{
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+			),
+		},
+		ClaimSize: "5Gi",
+	}
+	// TODO: strict checking of the file content once https://github.com/kubernetes/kubernetes/pull/71589
+	// or something similar is merged.
+	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, driver); err != nil {
+		return nil, errors.Wrap(err, filename)
+	}
+	return driver, nil
+}
+
+var _ testsuites.TestDriver = &driverDefinition{}
+
+// We have to implement the interface because dynamic PV may or may
+// not be supported. driverDefinition.SkipUnsupportedTest checks that
+// based on the actual driver definition.
+var _ testsuites.DynamicPVTestDriver = &driverDefinition{}
+
+// Same for snapshotting.
+var _ testsuites.SnapshottableTestDriver = &driverDefinition{}
+
+// runtime.DecodeInto needs a runtime.Object but doesn't do any
+// deserialization of it and therefore none of the methods below need
+// an implementation.
+var _ runtime.Object = &driverDefinition{}
+
+// DriverDefinition needs to be filled in via a .yaml or .json
+// file. It's methods then implement the TestDriver interface, using
+// nothing but the information in this struct.
+type driverDefinition struct {
+	// DriverInfo is the static information that the storage testsuite
+	// expects from a test driver. See test/e2e/storage/testsuites/testdriver.go
+	// for details. The only field with a non-zero default is the list of
+	// supported file systems (SupportedFsType): it is set so that tests using
+	// the default file system are enabled.
+	DriverInfo testsuites.DriverInfo
+
+	// ShortName is used to create unique names for test cases and test resources.
+	ShortName string
+
+	// StorageClass must be set to enable dynamic provisioning tests.
+	// The default is to not run those tests.
+	StorageClass struct {
+		// FromName set to true enables the usage of a storage
+		// class with DriverInfo.Name as provisioner and no
+		// parameters.
+		FromName bool
+
+		// FromFile is used only when FromName is false.  It
+		// loads a storage class from the given .yaml or .json
+		// file. File names are resolved by the
+		// framework.testfiles package, which typically means
+		// that they can be absolute or relative to the test
+		// suite's --repo-root parameter.
+		//
+		// This can be used when the storage class is meant to have
+		// additional parameters.
+		FromFile string
+	}
+
+	// SnapshotClass must be set to enable snapshotting tests.
+	// The default is to not run those tests.
+	SnapshotClass struct {
+		// FromName set to true enables the usage of a
+		// snapshotter class with DriverInfo.Name as provisioner.
+		FromName bool
+
+		// TODO (?): load from file
+	}
+
+	// ClaimSize defines the desired size of dynamically
+	// provisioned volumes. Default is "5GiB".
+	ClaimSize string
+
+	// ClientNodeName selects a specific node for scheduling test pods.
+	// Can be left empty. Most drivers should not need this and instead
+	// use topology to ensure that pods land on the right node(s).
+	ClientNodeName string
+}
+
+func (d *driverDefinition) DeepCopyObject() runtime.Object {
+	return nil
+}
+
+func (d *driverDefinition) GetObjectKind() schema.ObjectKind {
+	return nil
+}
+
+func (d *driverDefinition) GetDriverInfo() *testsuites.DriverInfo {
+	return &d.DriverInfo
+}
+
+func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+	supported := false
+	// TODO (?): add support for more volume types
+	switch pattern.VolType {
+	case testpatterns.DynamicPV:
+		if d.StorageClass.FromName || d.StorageClass.FromFile != "" {
+			supported = true
+		}
+	}
+	if !supported {
+		framework.Skipf("Driver %q does not support volume type %q - skipping", d.DriverInfo.Name, pattern.VolType)
+	}
+
+	supported = false
+	switch pattern.SnapshotType {
+	case "":
+		supported = true
+	case testpatterns.DynamicCreatedSnapshot:
+		if d.SnapshotClass.FromName {
+			supported = true
+		}
+	}
+	if !supported {
+		framework.Skipf("Driver %q does not support snapshot type %q - skipping", d.DriverInfo.Name, pattern.SnapshotType)
+	}
+}
+
+func (d *driverDefinition) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+	f := config.Framework
+
+	if d.StorageClass.FromName {
+		provisioner := d.DriverInfo.Name
+		parameters := map[string]string{}
+		ns := f.Namespace.Name
+		suffix := provisioner + "-sc"
+		if fsType != "" {
+			parameters["csi.storage.k8s.io/fstype"] = fsType
+		}
+
+		return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
+	}
+
+	items, err := f.LoadFromManifests(d.StorageClass.FromFile)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred(), "load storage class from %s", d.StorageClass.FromFile)
+	gomega.Expect(len(items)).To(gomega.Equal(1), "exactly one item from %s", d.StorageClass.FromFile)
+
+	err = f.PatchItems(items...)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred(), "patch items")
+
+	sc, ok := items[0].(*storagev1.StorageClass)
+	gomega.Expect(ok).To(gomega.BeTrue(), "storage class from %s", d.StorageClass.FromFile)
+	if fsType != "" {
+		if sc.Parameters == nil {
+			sc.Parameters = map[string]string{}
+		}
+		sc.Parameters["csi.storage.k8s.io/fstype"] = fsType
+	}
+	return sc
+}
+
+func (d *driverDefinition) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured {
+	if !d.SnapshotClass.FromName {
+		framework.Skipf("Driver %q does not support snapshotting - skipping", d.DriverInfo.Name)
+	}
+
+	snapshotter := config.GetUniqueDriverName()
+	parameters := map[string]string{}
+	ns := config.Framework.Namespace.Name
+	suffix := snapshotter + "-vsc"
+
+	return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
+}
+
+func (d *driverDefinition) GetClaimSize() string {
+	return d.ClaimSize
+}
+
+func (d *driverDefinition) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	config := &testsuites.PerTestConfig{
+		Driver:         d,
+		Prefix:         "external",
+		Framework:      f,
+		ClientNodeName: d.ClientNodeName,
+	}
+	return config, func() {}
+}

test/e2e/storage/external/external_test.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package external
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+)
+
+func TestDriverParameter(t *testing.T) {
+	expected := &driverDefinition{
+		DriverInfo: testsuites.DriverInfo{
+			Name: "foo.example.com",
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+			),
+		},
+		ShortName: "foo",
+		ClaimSize: "5Gi",
+	}
+	testcases := []struct {
+		name     string
+		filename string
+		err      string
+		expected *driverDefinition
+	}{
+		{
+			name:     "no such file",
+			filename: "no-such-file.yaml",
+			err:      "open no-such-file.yaml: no such file or directory",
+		},
+		{
+			name: "empty file name",
+			err:  "missing file name",
+		},
+		{
+			name:     "yaml",
+			filename: "testdata/driver.yaml",
+			expected: expected,
+		},
+		{
+			name:     "json",
+			filename: "testdata/driver.json",
+			expected: expected,
+		},
+	}
+
+	for _, testcase := range testcases {
+		actual, err := testDriverParameter{}.loadDriverDefinition(testcase.filename)
+		if testcase.err == "" {
+			assert.NoError(t, err, testcase.name)
+		} else {
+			if assert.Error(t, err, testcase.name) {
+				assert.Equal(t, testcase.err, err.Error())
+			}
+		}
+		if err == nil {
+			assert.Equal(t, testcase.expected, actual)
+		}
+	}
+}

test/e2e/storage/external/testdata/driver.json

@@ -0,0 +1 @@
+{"DriverInfo": {"Name": "foo.example.com"}, "ShortName": "foo"}

test/e2e/storage/external/testdata/driver.yaml

@@ -0,0 +1,3 @@
+DriverInfo:
+  Name: foo.example.com
+ShortName: foo

test/e2e/storage/testsuites/testdriver.go

@@ -125,7 +125,7 @@ const (
 
 // DriverInfo represents static information about a TestDriver.
 type DriverInfo struct {
-	Name       string // Name of the driver
+	Name       string // Name of the driver, aka the provisioner name.
 	FeatureTag string // FeatureTag for the driver
 
 	MaxFileSize          int64               // Max file size to be tested for this driver