storage e2e: verify CSIStorageCapacity publishing

Drivers need to opt into the new test. Depending on how the driver
describes its behavior, the check can be more specific. Currently it
distinguishes between getting any kind of information about the
storage class (i.e. assuming that capacity is not exhausted) and
getting one object per node (for local storage). Discovering nodes
only works for CSI drivers.

The immediate usage of this new test is for csi-driver-host-path with
the deployment for distributed provisioning and storage capacity
tracking. Periodic kubernetes-csi Prow and pre-merge jobs can run this
test.

The alternative would have been to write a test that manages the
deployment of the csi-driver-host-path driver itself, i.e. use the E2E
manifests. But that would have implied duplicating the
deployments (in-tree and in csi-driver-host-path) and then changing
kubernetes-csi Prow jobs to somehow run for in-tree driver definition
with newer components, something that currently isn't done. The test
then also wouldn't be applicable to out-of-tree driver deployments.

Yet another alternative would be to create a separate E2E test suite
either in csi-driver-host-path or external-provisioner. The advantage
of such an approach is that the test can be written exactly for the
expected behavior of a deployment and thus be more precise than the
generic version of the test in k/k. But this again wouldn't be
reusable for other drivers and also a lot of work to set up as no such
E2E test suite currently exists.

test/e2e/storage/framework/testdriver.go

@@ -144,7 +144,7 @@ func GetDriverTimeouts(driver TestDriver) *framework.TimeoutContext {
 // Capability represents a feature that a volume plugin supports
 type Capability string
 
-// Constants related to capability
+// Constants related to capabilities and behavior of the driver.
 const (
 	CapPersistence        Capability = "persistence"        // data is persisted across pod restarts
 	CapBlock              Capability = "block"              // raw block mode
@@ -166,6 +166,18 @@ const (
 	CapVolumeLimits        Capability = "volumeLimits"        // support volume limits (can be *very* slow)
 	CapSingleNodeVolume    Capability = "singleNodeVolume"    // support volume that can run on single node (like hostpath)
 	CapTopology            Capability = "topology"            // support topology
+
+	// The driver publishes storage capacity information: when the storage class
+	// for dynamic provisioning exists, the driver is expected to provide
+	// capacity information for it.
+	CapCapacity Capability = "capacity"
+
+	// The driver manages storage locally through CSI. TopologyKeys must be
+	// set and contain exactly one entry for distinguishing nodes.
+	// When the driver supports CapCapacity and the storage class
+	// for dynamic provisioning exists, the driver is expected to
+	// provider capacity information for it for each node.
+	CapCSILocalStorage Capability = "CSILocalStorage"
 )
 
 // DriverInfo represents static information about a TestDriver.

test/e2e/storage/testsuites/base.go

@@ -53,6 +53,7 @@ type migrationOpCheck struct {
 
 // BaseSuites is a list of storage test suites that work for in-tree and CSI drivers
 var BaseSuites = []func() storageframework.TestSuite{
+	InitCapacityTestSuite,
 	InitVolumesTestSuite,
 	InitVolumeIOTestSuite,
 	InitVolumeModeTestSuite,

test/e2e/storage/testsuites/capacity.go

@@ -0,0 +1,380 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/types"
+
+	storagev1 "k8s.io/api/storage/v1"
+	storagev1beta1 "k8s.io/api/storage/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
+	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
+	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+type capacityTestSuite struct {
+	tsInfo storageframework.TestSuiteInfo
+}
+
+// InitCustomCapacityTestSuite returns capacityTestSuite that implements TestSuite interface
+// using custom test patterns
+func InitCustomCapacityTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
+	return &capacityTestSuite{
+		tsInfo: storageframework.TestSuiteInfo{
+			Name:         "capacity",
+			TestPatterns: patterns,
+			SupportedSizeRange: e2evolume.SizeRange{
+				Min: "1Mi",
+			},
+		},
+	}
+}
+
+// InitCapacityTestSuite returns capacityTestSuite that implements TestSuite interface\
+// using test suite default patterns
+func InitCapacityTestSuite() storageframework.TestSuite {
+	patterns := []storageframework.TestPattern{
+		storageframework.DefaultFsDynamicPV,
+	}
+	return InitCustomCapacityTestSuite(patterns)
+}
+
+func (p *capacityTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
+	return p.tsInfo
+}
+
+func (p *capacityTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
+	// Check preconditions.
+	if pattern.VolType != storageframework.DynamicPV {
+		e2eskipper.Skipf("Suite %q does not support %v", p.tsInfo.Name, pattern.VolType)
+	}
+	dInfo := driver.GetDriverInfo()
+	if !dInfo.Capabilities[storageframework.CapCapacity] {
+		e2eskipper.Skipf("Driver %s doesn't publish storage capacity -- skipping", dInfo.Name)
+	}
+}
+
+func (p *capacityTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
+	var (
+		dInfo         = driver.GetDriverInfo()
+		dDriver       storageframework.DynamicPVTestDriver
+		driverCleanup func()
+		sc            *storagev1.StorageClass
+	)
+
+	// Beware that it also registers an AfterEach which renders f unusable. Any code using
+	// f must run inside an It or Context callback.
+	f := framework.NewFrameworkWithCustomTimeouts("capacity", storageframework.GetDriverTimeouts(driver))
+
+	init := func() {
+		dDriver, _ = driver.(storageframework.DynamicPVTestDriver)
+		// Now do the more expensive test initialization.
+		config, cleanup := driver.PrepareTest(f)
+		driverCleanup = cleanup
+		sc = dDriver.GetDynamicProvisionStorageClass(config, pattern.FsType)
+		if sc == nil {
+			e2eskipper.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
+		}
+	}
+
+	cleanup := func() {
+		err := storageutils.TryFunc(driverCleanup)
+		driverCleanup = nil
+		framework.ExpectNoError(err, "while cleaning up driver")
+	}
+
+	ginkgo.It("provides storage capacity information", func() {
+		init()
+		defer cleanup()
+
+		timeout := time.Minute
+		pollInterval := time.Second
+		matchSC := HaveCapacitiesForClass(sc.Name)
+		listAll := gomega.Eventually(func() (*storagev1beta1.CSIStorageCapacityList, error) {
+			return f.ClientSet.StorageV1beta1().CSIStorageCapacities("").List(context.Background(), metav1.ListOptions{})
+		}, timeout, pollInterval)
+
+		// If we have further information about what storage
+		// capacity information to expect from the driver,
+		// then we can make the check more specific. The baseline
+		// is that it provides some arbitrary capacity for the
+		// storage class.
+		matcher := matchSC
+		if dInfo.Capabilities[storageframework.CapCSILocalStorage] {
+			// Local storage. We expect one CSIStorageCapacity object per
+			// node for the storage class.
+			if len(dInfo.TopologyKeys) != 1 {
+				framework.Failf("need exactly one topology key for local storage, DriverInfo.TopologyKeys has: %v", dInfo.TopologyKeys)
+			}
+			matcher = HaveCapacitiesForClassAndNodes(f.ClientSet, sc.Provisioner, sc.Name, dInfo.TopologyKeys[0])
+		}
+
+		// Create storage class and wait for capacity information.
+		_, clearProvisionedStorageClass := SetupStorageClass(f.ClientSet, sc)
+		defer clearProvisionedStorageClass()
+		listAll.Should(MatchCapacities(matcher), "after creating storage class")
+
+		// Delete storage class again and wait for removal of storage capacity information.
+		clearProvisionedStorageClass()
+		listAll.ShouldNot(MatchCapacities(matchSC), "after deleting storage class")
+	})
+}
+
+func formatCapacities(capacities []storagev1beta1.CSIStorageCapacity) []string {
+	lines := []string{}
+	for _, capacity := range capacities {
+		lines = append(lines, fmt.Sprintf("   %+v", capacity))
+	}
+	return lines
+}
+
+// MatchCapacities runs some kind of check against *storagev1beta1.CSIStorageCapacityList.
+// In case of failure, all actual objects are appended to the failure message.
+func MatchCapacities(match types.GomegaMatcher) types.GomegaMatcher {
+	return matchCSIStorageCapacities{match: match}
+}
+
+type matchCSIStorageCapacities struct {
+	match types.GomegaMatcher
+}
+
+var _ types.GomegaMatcher = matchCSIStorageCapacities{}
+
+func (m matchCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
+	return m.match.Match(actual)
+}
+
+func (m matchCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
+	return m.match.FailureMessage(actual) + m.dump(actual)
+}
+
+func (m matchCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
+	return m.match.NegatedFailureMessage(actual) + m.dump(actual)
+}
+
+func (m matchCSIStorageCapacities) dump(actual interface{}) string {
+	capacities, ok := actual.(*storagev1beta1.CSIStorageCapacityList)
+	if !ok || capacities == nil {
+		return ""
+	}
+	lines := []string{"\n\nall CSIStorageCapacity objects:"}
+	for _, capacity := range capacities.Items {
+		lines = append(lines, fmt.Sprintf("%+v", capacity))
+	}
+	return strings.Join(lines, "\n")
+}
+
+// CapacityMatcher can be used to compose different matchers where one
+// adds additional checks for CSIStorageCapacity objects already checked
+// by another.
+type CapacityMatcher interface {
+	types.GomegaMatcher
+	// MatchedCapacities returns all CSICapacityObjects which were
+	// found during the preceding Match call.
+	MatchedCapacities() []storagev1beta1.CSIStorageCapacity
+}
+
+// HaveCapacitiesForClass filters all storage capacity objects in a *storagev1beta1.CSIStorageCapacityList
+// by storage class. Success is when when there is at least one.
+func HaveCapacitiesForClass(scName string) CapacityMatcher {
+	return &haveCSIStorageCapacities{scName: scName}
+}
+
+type haveCSIStorageCapacities struct {
+	scName             string
+	matchingCapacities []storagev1beta1.CSIStorageCapacity
+}
+
+var _ CapacityMatcher = &haveCSIStorageCapacities{}
+
+func (h *haveCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
+	capacities, ok := actual.(*storagev1beta1.CSIStorageCapacityList)
+	if !ok {
+		return false, fmt.Errorf("expected *storagev1beta1.CSIStorageCapacityList, got: %T", actual)
+	}
+	h.matchingCapacities = nil
+	for _, capacity := range capacities.Items {
+		if capacity.StorageClassName == h.scName {
+			h.matchingCapacities = append(h.matchingCapacities, capacity)
+		}
+	}
+	return len(h.matchingCapacities) > 0, nil
+}
+
+func (h *haveCSIStorageCapacities) MatchedCapacities() []storagev1beta1.CSIStorageCapacity {
+	return h.matchingCapacities
+}
+
+func (h *haveCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("no CSIStorageCapacity objects for storage class %q", h.scName)
+}
+
+func (h *haveCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("CSIStorageCapacity objects for storage class %q:\n%s",
+		h.scName,
+		strings.Join(formatCapacities(h.matchingCapacities), "\n"),
+	)
+}
+
+// HaveCapacitiesForClassAndNodes matches objects by storage class name. It finds
+// all nodes on which the driver runs and expects one object per node.
+func HaveCapacitiesForClassAndNodes(client kubernetes.Interface, driverName, scName, topologyKey string) CapacityMatcher {
+	return &haveLocalStorageCapacities{
+		client:      client,
+		driverName:  driverName,
+		match:       HaveCapacitiesForClass(scName),
+		topologyKey: topologyKey,
+	}
+}
+
+type haveLocalStorageCapacities struct {
+	client      kubernetes.Interface
+	driverName  string
+	match       CapacityMatcher
+	topologyKey string
+
+	matchSuccess          bool
+	topologyValues        []string
+	expectedCapacities    []storagev1beta1.CSIStorageCapacity
+	unexpectedCapacities  []storagev1beta1.CSIStorageCapacity
+	missingTopologyValues []string
+}
+
+var _ CapacityMatcher = &haveLocalStorageCapacities{}
+
+func (h *haveLocalStorageCapacities) Match(actual interface{}) (success bool, err error) {
+	h.topologyValues = nil
+	h.expectedCapacities = nil
+	h.unexpectedCapacities = nil
+	h.missingTopologyValues = nil
+
+	// First check with underlying matcher.
+	success, err = h.match.Match(actual)
+	h.matchSuccess = success
+	if !success || err != nil {
+		return
+	}
+
+	// Find all nodes on which the driver runs.
+	csiNodes, err := h.client.StorageV1().CSINodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return false, err
+	}
+	for _, csiNode := range csiNodes.Items {
+		for _, driver := range csiNode.Spec.Drivers {
+			if driver.Name != h.driverName {
+				continue
+			}
+			node, err := h.client.CoreV1().Nodes().Get(context.Background(), csiNode.Name, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+			value, ok := node.Labels[h.topologyKey]
+			if !ok || value == "" {
+				return false, fmt.Errorf("driver %q should run on node %q, but its topology label %q was not set",
+					h.driverName,
+					node.Name,
+					h.topologyKey)
+			}
+			h.topologyValues = append(h.topologyValues, value)
+			break
+		}
+	}
+	if len(h.topologyValues) == 0 {
+		return false, fmt.Errorf("driver %q not running on any node", h.driverName)
+	}
+
+	// Now check that for each topology value there is exactly one CSIStorageCapacity object.
+	remainingTopologyValues := map[string]bool{}
+	for _, value := range h.topologyValues {
+		remainingTopologyValues[value] = true
+	}
+	capacities := h.match.MatchedCapacities()
+	for _, capacity := range capacities {
+		if capacity.NodeTopology == nil ||
+			len(capacity.NodeTopology.MatchExpressions) > 0 ||
+			len(capacity.NodeTopology.MatchLabels) != 1 ||
+			!remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] {
+			h.unexpectedCapacities = append(h.unexpectedCapacities, capacity)
+			continue
+		}
+		remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] = false
+		h.expectedCapacities = append(h.expectedCapacities, capacity)
+	}
+
+	// Success is when there were no unexpected capacities and enough expected ones.
+	for value, remaining := range remainingTopologyValues {
+		if remaining {
+			h.missingTopologyValues = append(h.missingTopologyValues, value)
+		}
+	}
+	return len(h.unexpectedCapacities) == 0 && len(h.missingTopologyValues) == 0, nil
+}
+
+func (h *haveLocalStorageCapacities) MatchedCapacities() []storagev1beta1.CSIStorageCapacity {
+	return h.match.MatchedCapacities()
+}
+
+func (h *haveLocalStorageCapacities) FailureMessage(actual interface{}) (message string) {
+	if !h.matchSuccess {
+		return h.match.FailureMessage(actual)
+	}
+	var lines []string
+	if len(h.unexpectedCapacities) != 0 {
+		lines = append(lines, "unexpected CSIStorageCapacity objects:")
+		lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
+	}
+	if len(h.missingTopologyValues) != 0 {
+		lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
+			h.topologyKey, h.missingTopologyValues,
+		))
+	}
+	return strings.Join(lines, "\n")
+}
+
+func (h *haveLocalStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
+	if h.matchSuccess {
+		return h.match.NegatedFailureMessage(actual)
+	}
+	// It's not entirely clear whether negating this check is useful. Just dump all info that we have.
+	var lines []string
+	if len(h.expectedCapacities) != 0 {
+		lines = append(lines, "expected CSIStorageCapacity objects:")
+		lines = append(lines, formatCapacities(h.expectedCapacities)...)
+	}
+	if len(h.unexpectedCapacities) != 0 {
+		lines = append(lines, "unexpected CSIStorageCapacity objects:")
+		lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
+	}
+	if len(h.missingTopologyValues) != 0 {
+		lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
+			h.topologyKey, h.missingTopologyValues,
+		))
+	}
+	return strings.Join(lines, "\n")
+}

test/e2e/storage/testsuites/provisioning.go

@@ -337,7 +337,10 @@ func SetupStorageClass(
 			framework.ExpectNoError(err)
 			clearComputedStorageClass = func() {
 				framework.Logf("deleting storage class %s", computedStorageClass.Name)
-				framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(context.TODO(), computedStorageClass.Name, metav1.DeleteOptions{}))
+				err := client.StorageV1().StorageClasses().Delete(context.TODO(), computedStorageClass.Name, metav1.DeleteOptions{})
+				if err != nil && !apierrors.IsNotFound(err) {
+					framework.ExpectNoError(err, "delete storage class")
+				}
 			}
 		}
 	} else {