diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index bcf30cbf0a8..447caea58b4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -975,8 +975,8 @@ }, { "ImportPath": "github.com/container-storage-interface/spec/lib/go/csi", - "Comment": "v1.0.0", - "Rev": "ed0bb0e1557548aa028307f48728767cfe8f6345" + "Comment": "v1.1.0-rc1", + "Rev": "c751be1edc7952e7aac35720d5b34b669e48f113" }, { "ImportPath": "github.com/containerd/console", diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index e2bed5b725d..4188842916c 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -109,6 +109,11 @@ const ( // Ability to expand persistent volumes' file system without unmounting volumes. ExpandInUsePersistentVolumes utilfeature.Feature = "ExpandInUsePersistentVolumes" + // owner: @gnufied + // alpha: v1.14 + // Ability to expand CSI volumes + ExpandCSIVolumes utilfeature.Feature = "ExpandCSIVolumes" + // owner: @verb // alpha: v1.10 // @@ -450,6 +455,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS QOSReserved: {Default: false, PreRelease: utilfeature.Alpha}, ExpandPersistentVolumes: {Default: true, PreRelease: utilfeature.Beta}, ExpandInUsePersistentVolumes: {Default: false, PreRelease: utilfeature.Alpha}, + ExpandCSIVolumes: {Default: false, PreRelease: utilfeature.Alpha}, AttachVolumeLimit: {Default: true, PreRelease: utilfeature.Beta}, CPUManager: {Default: true, PreRelease: utilfeature.Beta}, CPUCFSQuotaPeriod: {Default: false, PreRelease: utilfeature.Alpha}, diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 360d3ab0b71..5ecec2342d3 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -542,7 +542,7 @@ func (asw *actualStateOfWorld) MarkFSResizeRequired( } volumePlugin, err := - asw.volumePluginMgr.FindExpandablePluginBySpec(podObj.volumeSpec) + asw.volumePluginMgr.FindNodeExpandablePluginBySpec(podObj.volumeSpec) if err != nil || volumePlugin == nil { // Log and continue processing klog.Errorf( diff --git a/pkg/volume/awsebs/aws_ebs.go b/pkg/volume/awsebs/aws_ebs.go index 8a98699473c..274ac79f2b1 100644 --- a/pkg/volume/awsebs/aws_ebs.go +++ b/pkg/volume/awsebs/aws_ebs.go @@ -317,12 +317,15 @@ func (plugin *awsElasticBlockStorePlugin) ExpandVolumeDevice( return awsVolume.ResizeDisk(volumeID, oldSize, newSize) } -func (plugin *awsElasticBlockStorePlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (plugin *awsElasticBlockStorePlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } -var _ volume.FSResizableVolumePlugin = &awsElasticBlockStorePlugin{} +var _ volume.NodeExpandableVolumePlugin = &awsElasticBlockStorePlugin{} var _ volume.ExpandableVolumePlugin = &awsElasticBlockStorePlugin{} var _ volume.VolumePluginWithAttachLimits = &awsElasticBlockStorePlugin{} diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index e9a850f9016..b5b040b4ab4 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -312,12 +312,15 @@ func (plugin *azureDataDiskPlugin) ExpandVolumeDevice( return diskController.ResizeDisk(spec.PersistentVolume.Spec.AzureDisk.DataDiskURI, oldSize, newSize) } -func (plugin *azureDataDiskPlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (plugin *azureDataDiskPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } -var _ volume.FSResizableVolumePlugin = &azureDataDiskPlugin{} +var _ volume.NodeExpandableVolumePlugin = &azureDataDiskPlugin{} func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 7240614c783..3347b46eccd 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -304,12 +304,15 @@ func (plugin *cinderPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resour return expandedSize, nil } -func (plugin *cinderPlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (plugin *cinderPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } -var _ volume.FSResizableVolumePlugin = &cinderPlugin{} +var _ volume.NodeExpandableVolumePlugin = &cinderPlugin{} func (plugin *cinderPlugin) RequiresFSResize() bool { return true diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index a51d3cb8a71..71e07796af6 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -10,6 +10,7 @@ go_library( "csi_mounter.go", "csi_plugin.go", "csi_util.go", + "expander.go", ], importpath = "k8s.io/kubernetes/pkg/volume/csi", visibility = ["//visibility:public"], @@ -22,6 +23,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", @@ -50,6 +52,7 @@ go_test( "csi_drivers_store_test.go", "csi_mounter_test.go", "csi_plugin_test.go", + "expander_test.go", ], embed = [":go_default_library"], deps = [ diff --git a/pkg/volume/csi/csi_client.go b/pkg/volume/csi/csi_client.go index 16a46b527d0..075123080b1 100644 --- a/pkg/volume/csi/csi_client.go +++ b/pkg/volume/csi/csi_client.go @@ -28,6 +28,7 @@ import ( csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -55,6 +56,7 @@ type csiClient interface { fsType string, mountOptions []string, ) error + NodeExpandVolume(ctx context.Context, volumeid, volumePath string, newSize resource.Quantity) (resource.Quantity, error) NodeUnpublishVolume( ctx context.Context, volID string, @@ -71,6 +73,7 @@ type csiClient interface { ) error NodeUnstageVolume(ctx context.Context, volID, stagingTargetPath string) error NodeSupportsStageUnstage(ctx context.Context) (bool, error) + NodeSupportsNodeExpand(ctx context.Context) (bool, error) } // Strongly typed address @@ -304,6 +307,41 @@ func (c *csiDriverClient) NodePublishVolume( } +func (c *csiDriverClient) NodeExpandVolume(ctx context.Context, volumeID, volumePath string, newSize resource.Quantity) (resource.Quantity, error) { + if c.nodeV1ClientCreator == nil { + return newSize, fmt.Errorf("version of CSI driver does not support volume expansion") + } + + if volumeID == "" { + return newSize, errors.New("missing volume id") + } + if volumePath == "" { + return newSize, errors.New("missing volume path") + } + + if newSize.Value() < 0 { + return newSize, errors.New("size can not be less than 0") + } + + nodeClient, closer, err := c.nodeV1ClientCreator(c.addr) + if err != nil { + return newSize, err + } + defer closer.Close() + + req := &csipbv1.NodeExpandVolumeRequest{ + VolumeId: volumeID, + VolumePath: volumePath, + CapacityRange: &csipbv1.CapacityRange{RequiredBytes: newSize.Value()}, + } + resp, err := nodeClient.NodeExpandVolume(ctx, req) + if err != nil { + return newSize, err + } + updatedQuantity := resource.NewQuantity(resp.CapacityBytes, resource.BinarySI) + return *updatedQuantity, nil +} + func (c *csiDriverClient) nodePublishVolumeV1( ctx context.Context, volID string, @@ -624,6 +662,41 @@ func (c *csiDriverClient) nodeUnstageVolumeV0(ctx context.Context, volID, stagin return err } +func (c *csiDriverClient) NodeSupportsNodeExpand(ctx context.Context) (bool, error) { + klog.V(4).Info(log("calling NodeGetCapabilities rpc to determine if Node has EXPAND_VOLUME capability")) + + if c.nodeV1ClientCreator != nil { + nodeClient, closer, err := c.nodeV1ClientCreator(c.addr) + if err != nil { + return false, err + } + defer closer.Close() + + req := &csipbv1.NodeGetCapabilitiesRequest{} + resp, err := nodeClient.NodeGetCapabilities(ctx, req) + if err != nil { + return false, err + } + + capabilities := resp.GetCapabilities() + + nodeExpandSet := false + if capabilities == nil { + return false, nil + } + for _, capability := range capabilities { + if capability.GetRpc().GetType() == csipbv1.NodeServiceCapability_RPC_EXPAND_VOLUME { + nodeExpandSet = true + } + } + return nodeExpandSet, nil + } else if c.nodeV0ClientCreator != nil { + return false, nil + } + return false, fmt.Errorf("failed to call NodeSupportsNodeExpand. Both nodeV1ClientCreator and nodeV0ClientCreator are nil") + +} + func (c *csiDriverClient) NodeSupportsStageUnstage(ctx context.Context) (bool, error) { klog.V(4).Info(log("calling NodeGetCapabilities rpc to determine if NodeSupportsStageUnstage")) diff --git a/pkg/volume/csi/csi_client_test.go b/pkg/volume/csi/csi_client_test.go index 78d70333277..8e5c05ba8fe 100644 --- a/pkg/volume/csi/csi_client_test.go +++ b/pkg/volume/csi/csi_client_test.go @@ -25,6 +25,7 @@ import ( csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/volume/csi/fake" ) @@ -40,6 +41,13 @@ func newFakeCsiDriverClient(t *testing.T, stagingCapable bool) *fakeCsiDriverCli } } +func newFakeCsiDriverClientWithExpansion(t *testing.T, stagingCapable bool, expansionSet bool) *fakeCsiDriverClient { + return &fakeCsiDriverClient{ + t: t, + nodeClient: fake.NewNodeClientWithExpansion(stagingCapable, expansionSet), + } +} + func (c *fakeCsiDriverClient) NodeGetInfo(ctx context.Context) ( nodeID string, maxVolumePerNode int64, @@ -144,6 +152,28 @@ func (c *fakeCsiDriverClient) NodeUnstageVolume(ctx context.Context, volID, stag return err } +func (c *fakeCsiDriverClient) NodeSupportsNodeExpand(ctx context.Context) (bool, error) { + c.t.Log("calling fake.NodeSupportsNodeExpand...") + req := &csipbv1.NodeGetCapabilitiesRequest{} + + resp, err := c.nodeClient.NodeGetCapabilities(ctx, req) + if err != nil { + return false, err + } + + capabilities := resp.GetCapabilities() + + if capabilities == nil { + return false, nil + } + for _, capability := range capabilities { + if capability.GetRpc().GetType() == csipbv1.NodeServiceCapability_RPC_EXPAND_VOLUME { + return true, nil + } + } + return false, nil +} + func (c *fakeCsiDriverClient) NodeSupportsStageUnstage(ctx context.Context) (bool, error) { c.t.Log("calling fake.NodeGetCapabilities for NodeSupportsStageUnstage...") req := &csipbv1.NodeGetCapabilitiesRequest{} @@ -166,10 +196,29 @@ func (c *fakeCsiDriverClient) NodeSupportsStageUnstage(ctx context.Context) (boo return stageUnstageSet, nil } +func (c *fakeCsiDriverClient) NodeExpandVolume(ctx context.Context, volumeid, volumePath string, newSize resource.Quantity) (resource.Quantity, error) { + c.t.Log("calling fake.NodeExpandVolume") + req := &csipbv1.NodeExpandVolumeRequest{ + VolumeId: volumeid, + VolumePath: volumePath, + CapacityRange: &csipbv1.CapacityRange{RequiredBytes: newSize.Value()}, + } + resp, err := c.nodeClient.NodeExpandVolume(ctx, req) + if err != nil { + return newSize, err + } + updatedQuantity := resource.NewQuantity(resp.CapacityBytes, resource.BinarySI) + return *updatedQuantity, nil +} + func setupClient(t *testing.T, stageUnstageSet bool) csiClient { return newFakeCsiDriverClient(t, stageUnstageSet) } +func setupClientWithExpansion(t *testing.T, stageUnstageSet bool, expansionSet bool) csiClient { + return newFakeCsiDriverClientWithExpansion(t, stageUnstageSet, expansionSet) +} + func checkErr(t *testing.T, expectedAnError bool, actualError error) { t.Helper() @@ -415,3 +464,59 @@ func TestClientNodeUnstageVolume(t *testing.T) { } } } + +func TestNodeExpandVolume(t *testing.T) { + testCases := []struct { + name string + volID string + volumePath string + newSize resource.Quantity + mustFail bool + err error + }{ + { + name: "with all correct values", + volID: "vol-abcde", + volumePath: "/foo/bar", + newSize: resource.MustParse("10Gi"), + mustFail: false, + }, + { + name: "with missing volume-id", + volumePath: "/foo/bar", + newSize: resource.MustParse("10Gi"), + mustFail: true, + }, + { + name: "with missing volume path", + volID: "vol-1234", + newSize: resource.MustParse("10Gi"), + mustFail: true, + }, + { + name: "with invalid quantity", + volID: "vol-1234", + volumePath: "/foo/bar", + newSize: *resource.NewQuantity(-10, resource.DecimalSI), + mustFail: true, + }, + } + + for _, tc := range testCases { + t.Logf("Running test cases : %s", tc.name) + fakeCloser := fake.NewCloser(t) + client := &csiDriverClient{ + driverName: "Fake Driver Name", + nodeV1ClientCreator: func(addr csiAddr) (csipbv1.NodeClient, io.Closer, error) { + nodeClient := fake.NewNodeClient(false /* stagingCapable */) + nodeClient.SetNextError(tc.err) + return nodeClient, fakeCloser, nil + }, + } + _, err := client.NodeExpandVolume(context.Background(), tc.volID, tc.volumePath, tc.newSize) + checkErr(t, tc.mustFail, err) + if !tc.mustFail { + fakeCloser.Check() + } + } +} diff --git a/pkg/volume/csi/expander.go b/pkg/volume/csi/expander.go new file mode 100644 index 00000000000..265d5e54535 --- /dev/null +++ b/pkg/volume/csi/expander.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + api "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/volume" +) + +var _ volume.NodeExpandableVolumePlugin = &csiPlugin{} + +func (c *csiPlugin) RequiresFSResize() bool { + // We could check plugin's node capability but we instead are going to rely on + // NodeExpand to do the right thing and return early if plugin does not have + // node expansion capability. + if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandCSIVolumes) { + klog.V(4).Infof("Resizing is not enabled for CSI volume") + return false + } + return true +} + +func (c *csiPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + klog.V(4).Infof(log("Expander.NodeExpand(%s)", resizeOptions.DeviceMountPath)) + csiSource, err := getCSISourceFromSpec(resizeOptions.VolumeSpec) + if err != nil { + klog.Error(log("Expander.NodeExpand failed to get CSI persistent source: %v", err)) + return false, err + } + + csClient, err := newCsiDriverClient(csiDriverName(csiSource.Driver)) + if err != nil { + return false, err + } + + return c.nodeExpandWithClient(resizeOptions, csiSource, csClient) +} + +func (c *csiPlugin) nodeExpandWithClient( + resizeOptions volume.NodeResizeOptions, + csiSource *api.CSIPersistentVolumeSource, + csClient csiClient) (bool, error) { + driverName := csiSource.Driver + + ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) + defer cancel() + + nodeExpandSet, err := csClient.NodeSupportsNodeExpand(ctx) + if err != nil { + return false, fmt.Errorf("Expander.NodeExpand failed to check if node supports expansion : %v", err) + } + + if !nodeExpandSet { + return false, fmt.Errorf("Expander.NodeExpand found CSI plugin %s/%s to not support node expansion", c.GetPluginName(), driverName) + } + + // Check whether "STAGE_UNSTAGE_VOLUME" is set + stageUnstageSet, err := csClient.NodeSupportsStageUnstage(ctx) + if err != nil { + return false, fmt.Errorf("Expander.NodeExpand failed to check if plugins supports stage_unstage %v", err) + } + + // if plugin does not support STAGE_UNSTAGE but CSI volume path is staged + // it must mean this was placeholder staging performed by k8s and not CSI staging + // in which case we should return from here so as volume can be node published + // before we can resize + if !stageUnstageSet && resizeOptions.CSIVolumePhase == volume.CSIVolumeStaged { + return false, nil + } + + _, err = csClient.NodeExpandVolume(ctx, csiSource.VolumeHandle, resizeOptions.DeviceMountPath, resizeOptions.NewSize) + if err != nil { + return false, fmt.Errorf("Expander.NodeExpand failed to expand the volume : %v", err) + } + return true, nil +} diff --git a/pkg/volume/csi/expander_test.go b/pkg/volume/csi/expander_test.go new file mode 100644 index 00000000000..0bd9f128e20 --- /dev/null +++ b/pkg/volume/csi/expander_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "os" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kubernetes/pkg/volume" +) + +func TestNodeExpand(t *testing.T) { + tests := []struct { + name string + nodeExpansion bool + nodeStageSet bool + volumePhase volume.CSIVolumePhaseType + success bool + }{ + { + name: "when node expansion is not set", + success: false, + }, + { + name: "when nodeExpansion=on, nodeStage=on, volumePhase=staged", + nodeExpansion: true, + nodeStageSet: true, + volumePhase: volume.CSIVolumeStaged, + success: true, + }, + { + name: "when nodeExpansion=on, nodeStage=off, volumePhase=staged", + nodeExpansion: true, + volumePhase: volume.CSIVolumeStaged, + success: false, + }, + { + name: "when nodeExpansion=on, nodeStage=on, volumePhase=published", + nodeExpansion: true, + nodeStageSet: true, + volumePhase: volume.CSIVolumePublished, + success: true, + }, + { + name: "when nodeExpansion=on, nodeStage=off, volumePhase=published", + nodeExpansion: true, + volumePhase: volume.CSIVolumePublished, + success: true, + }, + } + for _, tc := range tests { + plug, tmpDir := newTestPlugin(t, nil) + defer os.RemoveAll(tmpDir) + + spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "expandable", "test-vol"), false) + + newSize, _ := resource.ParseQuantity("20Gi") + + resizeOptions := volume.NodeResizeOptions{ + VolumeSpec: spec, + NewSize: newSize, + DeviceMountPath: "/foo/bar", + CSIVolumePhase: tc.volumePhase, + } + csiSource, _ := getCSISourceFromSpec(resizeOptions.VolumeSpec) + + csClient := setupClientWithExpansion(t, tc.nodeStageSet, tc.nodeExpansion) + + ok, err := plug.nodeExpandWithClient(resizeOptions, csiSource, csClient) + if ok != tc.success { + if err != nil { + t.Errorf("For %s : expected %v got %v with %v", tc.name, tc.success, ok, err) + } else { + t.Errorf("For %s : expected %v got %v", tc.name, tc.success, ok) + } + + } + } +} diff --git a/pkg/volume/csi/fake/fake_client.go b/pkg/volume/csi/fake/fake_client.go index c9ed5e6b9ab..f674fdd4402 100644 --- a/pkg/volume/csi/fake/fake_client.go +++ b/pkg/volume/csi/fake/fake_client.go @@ -67,6 +67,7 @@ type NodeClient struct { nodePublishedVolumes map[string]CSIVolume nodeStagedVolumes map[string]CSIVolume stageUnstageSet bool + expansionSet bool nodeGetInfoResp *csipb.NodeGetInfoResponse nextErr error } @@ -80,6 +81,15 @@ func NewNodeClient(stageUnstageSet bool) *NodeClient { } } +func NewNodeClientWithExpansion(stageUnstageSet bool, expansionSet bool) *NodeClient { + return &NodeClient{ + nodePublishedVolumes: make(map[string]CSIVolume), + nodeStagedVolumes: make(map[string]CSIVolume), + stageUnstageSet: stageUnstageSet, + expansionSet: expansionSet, + } +} + // SetNextError injects next expected error func (f *NodeClient) SetNextError(err error) { f.nextErr = err @@ -195,6 +205,29 @@ func (f *NodeClient) NodeUnstageVolume(ctx context.Context, req *csipb.NodeUnsta return &csipb.NodeUnstageVolumeResponse{}, nil } +// NodeExpandVolume implements csi method +func (f *NodeClient) NodeExpandVolume(ctx context.Context, req *csipb.NodeExpandVolumeRequest, opts ...grpc.CallOption) (*csipb.NodeExpandVolumeResponse, error) { + if f.nextErr != nil { + return nil, f.nextErr + } + + if req.GetVolumeId() == "" { + return nil, errors.New("missing volume id") + } + if req.GetVolumePath() == "" { + return nil, errors.New("missing volume path") + } + + if req.GetCapacityRange().RequiredBytes <= 0 { + return nil, errors.New("required bytes should be greater than 0") + } + + resp := &csipb.NodeExpandVolumeResponse{ + CapacityBytes: req.GetCapacityRange().RequiredBytes, + } + return resp, nil +} + // NodeGetId implements csi method func (f *NodeClient) NodeGetInfo(ctx context.Context, in *csipb.NodeGetInfoRequest, opts ...grpc.CallOption) (*csipb.NodeGetInfoResponse, error) { if f.nextErr != nil { @@ -206,20 +239,27 @@ func (f *NodeClient) NodeGetInfo(ctx context.Context, in *csipb.NodeGetInfoReque // NodeGetCapabilities implements csi method func (f *NodeClient) NodeGetCapabilities(ctx context.Context, in *csipb.NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*csipb.NodeGetCapabilitiesResponse, error) { resp := &csipb.NodeGetCapabilitiesResponse{ - Capabilities: []*csipb.NodeServiceCapability{ - { - Type: &csipb.NodeServiceCapability_Rpc{ - Rpc: &csipb.NodeServiceCapability_RPC{ - Type: csipb.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - }, - }, - }, - }, + Capabilities: []*csipb.NodeServiceCapability{}, } if f.stageUnstageSet { - return resp, nil + resp.Capabilities = append(resp.Capabilities, &csipb.NodeServiceCapability{ + Type: &csipb.NodeServiceCapability_Rpc{ + Rpc: &csipb.NodeServiceCapability_RPC{ + Type: csipb.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }) } - return nil, nil + if f.expansionSet { + resp.Capabilities = append(resp.Capabilities, &csipb.NodeServiceCapability{ + Type: &csipb.NodeServiceCapability_Rpc{ + Rpc: &csipb.NodeServiceCapability_RPC{ + Type: csipb.NodeServiceCapability_RPC_EXPAND_VOLUME, + }, + }, + }) + } + return resp, nil } // NodeGetVolumeStats implements csi method diff --git a/pkg/volume/flexvolume/expander-defaults.go b/pkg/volume/flexvolume/expander-defaults.go index 4a33e184e38..ec4fd29f748 100644 --- a/pkg/volume/flexvolume/expander-defaults.go +++ b/pkg/volume/flexvolume/expander-defaults.go @@ -36,10 +36,13 @@ func (e *expanderDefaults) ExpandVolumeDevice(spec *volume.Spec, newSize resourc return newSize, nil } -// the defaults for ExpandFS return a generic resize indicator that will trigger the operation executor to go ahead with +// the defaults for NodeExpand return a generic resize indicator that will trigger the operation executor to go ahead with // generic filesystem resize -func (e *expanderDefaults) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - klog.Warning(logPrefix(e.plugin), "using default filesystem resize for volume ", spec.Name(), ", at ", devicePath) - _, err := util.GenericResizeFS(e.plugin.host, e.plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (e *expanderDefaults) NodeExpand(rsOpt volume.NodeResizeOptions) (bool, error) { + klog.Warning(logPrefix(e.plugin), "using default filesystem resize for volume ", rsOpt.VolumeSpec.Name(), ", at ", rsOpt.DevicePath) + _, err := util.GenericResizeFS(e.plugin.host, e.plugin.GetPluginName(), rsOpt.DevicePath, rsOpt.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } diff --git a/pkg/volume/flexvolume/expander.go b/pkg/volume/flexvolume/expander.go index 345bad26d7b..7ad06df8259 100644 --- a/pkg/volume/flexvolume/expander.go +++ b/pkg/volume/flexvolume/expander.go @@ -18,9 +18,10 @@ package flexvolume import ( "fmt" + "strconv" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/volume" - "strconv" ) func (plugin *flexVolumePlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) { @@ -42,26 +43,29 @@ func (plugin *flexVolumePlugin) ExpandVolumeDevice(spec *volume.Spec, newSize re return newSize, err } -func (plugin *flexVolumePlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, newSize, oldSize resource.Quantity) error { +func (plugin *flexVolumePlugin) NodeExpand(rsOpt volume.NodeResizeOptions) (bool, error) { // This method is called after we spec.PersistentVolume.Spec.Capacity // has been updated to the new size. The underlying driver thus sees // the _new_ (requested) size and can find out the _current_ size from // its underlying storage implementation - if spec.PersistentVolume == nil { - return fmt.Errorf("PersistentVolume not found for spec: %s", spec.Name()) + if rsOpt.VolumeSpec.PersistentVolume == nil { + return false, fmt.Errorf("PersistentVolume not found for spec: %s", rsOpt.VolumeSpec.Name()) } call := plugin.NewDriverCall(expandFSCmd) - call.AppendSpec(spec, plugin.host, nil) - call.Append(devicePath) - call.Append(deviceMountPath) - call.Append(strconv.FormatInt(newSize.Value(), 10)) - call.Append(strconv.FormatInt(oldSize.Value(), 10)) + call.AppendSpec(rsOpt.VolumeSpec, plugin.host, nil) + call.Append(rsOpt.DevicePath) + call.Append(rsOpt.DeviceMountPath) + call.Append(strconv.FormatInt(rsOpt.NewSize.Value(), 10)) + call.Append(strconv.FormatInt(rsOpt.OldSize.Value(), 10)) _, err := call.Run() if isCmdNotSupportedErr(err) { - return newExpanderDefaults(plugin).ExpandFS(spec, devicePath, deviceMountPath, newSize, oldSize) + return newExpanderDefaults(plugin).NodeExpand(rsOpt) } - return err + if err != nil { + return false, err + } + return true, nil } diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index e907485a0f9..c0a95af96bd 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -56,7 +56,7 @@ type flexVolumeAttachablePlugin struct { var _ volume.AttachableVolumePlugin = &flexVolumeAttachablePlugin{} var _ volume.PersistentVolumePlugin = &flexVolumePlugin{} -var _ volume.FSResizableVolumePlugin = &flexVolumePlugin{} +var _ volume.NodeExpandableVolumePlugin = &flexVolumePlugin{} var _ volume.ExpandableVolumePlugin = &flexVolumePlugin{} var _ volume.DeviceMountableVolumePlugin = &flexVolumeAttachablePlugin{} diff --git a/pkg/volume/gcepd/gce_pd.go b/pkg/volume/gcepd/gce_pd.go index 01a66a2d3c3..b48fc283d5c 100644 --- a/pkg/volume/gcepd/gce_pd.go +++ b/pkg/volume/gcepd/gce_pd.go @@ -290,12 +290,15 @@ func (plugin *gcePersistentDiskPlugin) ExpandVolumeDevice( return updatedQuantity, nil } -func (plugin *gcePersistentDiskPlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (plugin *gcePersistentDiskPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + _, err := util.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } -var _ volume.FSResizableVolumePlugin = &gcePersistentDiskPlugin{} +var _ volume.NodeExpandableVolumePlugin = &gcePersistentDiskPlugin{} func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 76780a5f6f1..c26bc9a4f28 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -46,6 +46,9 @@ type ProbeEvent struct { Op ProbeOperation // The operation to the plugin } +// CSIVolumePhaseType stores information about CSI volume path. +type CSIVolumePhaseType string + const ( // Common parameter which can be specified in StorageClass to specify the desired FSType // Provisioners SHOULD implement support for this if they are block device based @@ -55,6 +58,8 @@ const ( ProbeAddOrUpdate ProbeOperation = 1 << iota ProbeRemove + CSIVolumeStaged CSIVolumePhaseType = "staged" + CSIVolumePublished CSIVolumePhaseType = "published" ) // VolumeOptions contains option information about a volume. @@ -85,6 +90,26 @@ type VolumeOptions struct { Parameters map[string]string } +// NodeResizeOptions contain options to be passed for node expansion. +type NodeResizeOptions struct { + VolumeSpec *Spec + + // DevicePath - location of actual device on the node. In case of CSI + // this just could be volumeID + DevicePath string + + // DeviceMountPath location where device is mounted on the node. If volume type + // is attachable - this would be global mount path otherwise + // it would be location where volume was mounted for the pod + DeviceMountPath string + + NewSize resource.Quantity + OldSize resource.Quantity + + // CSIVolumePhase contains volume phase on the node + CSIVolumePhase CSIVolumePhaseType +} + type DynamicPluginProber interface { Init() error @@ -219,18 +244,20 @@ type DeviceMountableVolumePlugin interface { } // ExpandableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that can be -// expanded +// expanded via control-plane ExpandVolumeDevice call. type ExpandableVolumePlugin interface { VolumePlugin ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) RequiresFSResize() bool } -// FSResizableVolumePlugin is an extension of ExpandableVolumePlugin and is used for volumes (flex) -// that require extra steps on nodes for expansion to complete -type FSResizableVolumePlugin interface { - ExpandableVolumePlugin - ExpandFS(spec *Spec, devicePath, deviceMountPath string, newSize, oldSize resource.Quantity) error +// NodeExpandableVolumePlugin is an expanded interface of VolumePlugin and is used for volumes that +// require expansion on the node via NodeExpand call. +type NodeExpandableVolumePlugin interface { + VolumePlugin + RequiresFSResize() bool + // NodeExpand expands volume on given deviceMountPath and returns true if resize is successful. + NodeExpand(resizeOptions NodeResizeOptions) (bool, error) } // VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of @@ -951,26 +978,26 @@ func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugi return nil, nil } -// FindFSResizablePluginBySpec fetches a persistent volume plugin by spec -func (pm *VolumePluginMgr) FindFSResizablePluginBySpec(spec *Spec) (FSResizableVolumePlugin, error) { +// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec +func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) { volumePlugin, err := pm.FindPluginBySpec(spec) if err != nil { return nil, err } - if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok { + if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { return fsResizablePlugin, nil } return nil, nil } -// FindFSResizablePluginByName fetches a persistent volume plugin by name -func (pm *VolumePluginMgr) FindFSResizablePluginByName(name string) (FSResizableVolumePlugin, error) { +// FindNodeExpandablePluginByName fetches a persistent volume plugin by name +func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) { volumePlugin, err := pm.FindPluginByName(name) if err != nil { return nil, err } - if fsResizablePlugin, ok := volumePlugin.(FSResizableVolumePlugin); ok { + if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { return fsResizablePlugin, nil } diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index d62dcbc1740..89df8315c73 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -201,12 +201,15 @@ func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource. } } -func (plugin *rbdPlugin) ExpandFS(spec *volume.Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - _, err := volutil.GenericResizeFS(plugin.host, plugin.GetPluginName(), devicePath, deviceMountPath) - return err +func (plugin *rbdPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { + _, err := volutil.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath) + if err != nil { + return false, err + } + return true, nil } -var _ volume.FSResizableVolumePlugin = &rbdPlugin{} +var _ volume.NodeExpandableVolumePlugin = &rbdPlugin{} func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { return expander.manager.ExpandImage(expander, oldSize, newSize) diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 0d3a6f66caf..b7b5396d2e5 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -280,7 +280,7 @@ var _ ProvisionableVolumePlugin = &FakeVolumePlugin{} var _ AttachableVolumePlugin = &FakeVolumePlugin{} var _ VolumePluginWithAttachLimits = &FakeVolumePlugin{} var _ DeviceMountableVolumePlugin = &FakeVolumePlugin{} -var _ FSResizableVolumePlugin = &FakeVolumePlugin{} +var _ NodeExpandableVolumePlugin = &FakeVolumePlugin{} func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume { volumeList := *list @@ -537,8 +537,8 @@ func (plugin *FakeVolumePlugin) RequiresFSResize() bool { return true } -func (plugin *FakeVolumePlugin) ExpandFS(spec *Spec, devicePath, deviceMountPath string, _, _ resource.Quantity) error { - return nil +func (plugin *FakeVolumePlugin) NodeExpand(resizeOptions NodeResizeOptions) (bool, error) { + return true, nil } func (plugin *FakeVolumePlugin) GetVolumeLimits() (map[string]int64, error) { diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index d2ed9edc690..91e4825329d 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -598,6 +598,12 @@ func (og *operationGenerator) GenerateMountVolumeFunc( klog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) } + var resizeDone bool + var resizeError error + resizeOptions := volume.NodeResizeOptions{ + DevicePath: devicePath, + } + if volumeDeviceMounter != nil { deviceMountPath, err := volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec) @@ -626,12 +632,16 @@ func (og *operationGenerator) GenerateMountVolumeFunc( return volumeToMount.GenerateError("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr) } + resizeOptions.DeviceMountPath = deviceMountPath + resizeOptions.CSIVolumePhase = volume.CSIVolumeStaged + // resizeFileSystem will resize the file system if user has requested a resize of // underlying persistent volume and is allowed to do so. - resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePluginName) + resizeDone, resizeError = og.resizeFileSystem(volumeToMount, resizeOptions, volumePluginName) - if resizeSimpleError != nil || resizeDetailedError != nil { - return resizeSimpleError, resizeDetailedError + if resizeError != nil { + klog.Errorf("MountVolume.resizeFileSystem failed with %v", resizeError) + return volumeToMount.GenerateError("MountVolume.MountDevice failed while expanding volume", resizeError) } } @@ -657,6 +667,20 @@ func (og *operationGenerator) GenerateMountVolumeFunc( verbosity = klog.Level(4) } klog.V(verbosity).Infof(detailedMsg) + resizeOptions.DeviceMountPath = volumeMounter.GetPath() + resizeOptions.CSIVolumePhase = volume.CSIVolumePublished + + // We need to call resizing here again in case resizing was not done during device mount. There could be + // two reasons of that: + // - Volume does not support DeviceMounter interface. + // - In case of CSI the volume does not have node stage_unstage capability. + if !resizeDone { + resizeDone, resizeError = og.resizeFileSystem(volumeToMount, resizeOptions, volumePluginName) + if resizeError != nil { + klog.Errorf("MountVolume.resizeFileSystem failed with %v", resizeError) + return volumeToMount.GenerateError("MountVolume.Setup failed while expanding volume", resizeError) + } + } // Update actual state of world markVolMountedErr := actualStateOfWorld.MarkVolumeAsMounted( @@ -689,15 +713,15 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } } -func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) { +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, rsOpts volume.NodeResizeOptions, pluginName string) (bool, error) { if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { klog.V(4).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) - return nil, nil + return true, nil } // Get expander, if possible expandableVolumePlugin, _ := - og.volumePluginMgr.FindFSResizablePluginBySpec(volumeToMount.VolumeSpec) + og.volumePluginMgr.FindNodeExpandablePluginBySpec(volumeToMount.VolumeSpec) if expandableVolumePlugin != nil && expandableVolumePlugin.RequiresFSResize() && @@ -706,7 +730,7 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(pv.Spec.ClaimRef.Name, metav1.GetOptions{}) if err != nil { // Return error rather than leave the file system un-resized, caller will log and retry - return volumeToMount.GenerateError("MountVolume.resizeFileSystem get PVC failed", err) + return false, fmt.Errorf("MountVolume.resizeFileSystem get PVC failed : %v", err) } pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage] @@ -719,10 +743,20 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem failed", "requested read-only file system") klog.Warningf(detailedMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) - return nil, nil + return true, nil } - if resizeErr := expandableVolumePlugin.ExpandFS(volumeToMount.VolumeSpec, devicePath, deviceMountPath, pvSpecCap, pvcStatusCap); resizeErr != nil { - return volumeToMount.GenerateError("MountVolume.resizeFileSystem failed", resizeErr) + rsOpts.VolumeSpec = volumeToMount.VolumeSpec + rsOpts.NewSize = pvSpecCap + rsOpts.OldSize = pvcStatusCap + resizeDone, resizeErr := expandableVolumePlugin.NodeExpand(rsOpts) + if resizeErr != nil { + return false, fmt.Errorf("MountVolume.resizeFileSystem failed : %v", resizeErr) + } + // Volume resizing is not done but it did not error out. This could happen if a CSI volume + // does not have node stage_unstage capability but was asked to resize the volume before + // node publish. In which case - we must retry resizing after node publish. + if !resizeDone { + return false, nil } simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem succeeded", "") og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg) @@ -731,12 +765,12 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi err = util.MarkFSResizeFinished(pvc, pv.Spec.Capacity, og.kubeClient) if err != nil { // On retry, resizeFileSystem will be called again but do nothing - return volumeToMount.GenerateError("MountVolume.resizeFileSystem update PVC status failed", err) + return false, fmt.Errorf("MountVolume.resizeFileSystem update PVC status failed : %v", err) } - return nil, nil + return true, nil } } - return nil, nil + return true, nil } func (og *operationGenerator) GenerateUnmountVolumeFunc( @@ -1446,41 +1480,58 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("VolumeFSResize.FindPluginBySpec failed", err) } - attachableVolumePlugin, err := - og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec) - if err != nil || attachableVolumePlugin == nil { - if attachableVolumePlugin == nil { - err = fmt.Errorf("AttachableVolumePlugin is nil") - } - return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("VolumeFSResize.FindAttachablePluginBySpec failed", err) - } - - volumeAttacher, err := attachableVolumePlugin.NewAttacher() - if err != nil || volumeAttacher == nil { - if volumeAttacher == nil { - err = fmt.Errorf("VolumeAttacher is nil") - } - return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("VolumeFSResize.NewAttacher failed", err) - } - - deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("VolumeFSResize.GetDeviceMountPath failed", err) - } - fsResizeFunc := func() (error, error) { - resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, volumeToMount.DevicePath, deviceMountPath, volumePlugin.GetPluginName()) + var resizeDone bool + var simpleErr, detailedErr error + resizeOptions := volume.NodeResizeOptions{ + VolumeSpec: volumeToMount.VolumeSpec, + } - if resizeSimpleError != nil || resizeDetailedError != nil { - return resizeSimpleError, resizeDetailedError + attachableVolumePlugin, _ := + og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec) + + if attachableVolumePlugin != nil { + volumeAttacher, _ := attachableVolumePlugin.NewAttacher() + if volumeAttacher != nil { + resizeOptions.CSIVolumePhase = volume.CSIVolumeStaged + resizeOptions.DevicePath = volumeToMount.DevicePath + dmp, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) + if err != nil { + return volumeToMount.GenerateError("VolumeFSResize.GetDeviceMountPath failed", err) + } + resizeOptions.DeviceMountPath = dmp + resizeDone, simpleErr, detailedErr = og.doOnlineExpansion(volumeToMount, actualStateOfWorld, resizeOptions, volumePlugin.GetPluginName()) + if simpleErr != nil || detailedErr != nil { + return simpleErr, detailedErr + } + if resizeDone { + return nil, nil + } + } } - markFSResizedErr := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.PodName, volumeToMount.VolumeName) - if markFSResizedErr != nil { - // On failure, return error. Caller will log and retry. - return volumeToMount.GenerateError("VolumeFSResize.MarkVolumeAsResized failed", markFSResizedErr) + // if we are here that means volume plugin does not support attach interface + volumeMounter, newMounterErr := volumePlugin.NewMounter( + volumeToMount.VolumeSpec, + volumeToMount.Pod, + volume.VolumeOptions{}) + if newMounterErr != nil { + return volumeToMount.GenerateError("VolumeFSResize.NewMounter initialization failed", newMounterErr) } - return nil, nil + + resizeOptions.DeviceMountPath = volumeMounter.GetPath() + resizeOptions.CSIVolumePhase = volume.CSIVolumePublished + resizeDone, simpleErr, detailedErr = og.doOnlineExpansion(volumeToMount, actualStateOfWorld, resizeOptions, volumePlugin.GetPluginName()) + if simpleErr != nil || detailedErr != nil { + return simpleErr, detailedErr + } + if resizeDone { + return nil, nil + } + // This is a placeholder error - we should NEVER reach here. + err := fmt.Errorf("volume resizing failed for unknown reason") + return volumeToMount.GenerateError("VolumeFSResize.resizeFileSystem failed to resize volume", err) } + eventRecorderFunc := func(err *error) { if *err != nil { og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.VolumeResizeFailed, (*err).Error()) @@ -1494,6 +1545,28 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( }, nil } +func (og *operationGenerator) doOnlineExpansion(volumeToMount VolumeToMount, + actualStateOfWorld ActualStateOfWorldMounterUpdater, + resizeOptions volume.NodeResizeOptions, + pluginName string) (bool, error, error) { + resizeDone, err := og.resizeFileSystem(volumeToMount, resizeOptions, pluginName) + if err != nil { + klog.Errorf("VolumeFSResize.resizeFileSystem failed : %v", err) + e1, e2 := volumeToMount.GenerateError("VolumeFSResize.resizeFileSystem failed", err) + return false, e1, e2 + } + if resizeDone { + markFSResizedErr := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.PodName, volumeToMount.VolumeName) + if markFSResizedErr != nil { + // On failure, return error. Caller will log and retry. + e1, e2 := volumeToMount.GenerateError("VolumeFSResize.MarkVolumeAsResized failed", markFSResizedErr) + return false, e1, e2 + } + return true, nil, nil + } + return false, nil, nil +} + func checkMountOptionSupport(og *operationGenerator, volumeToMount VolumeToMount, plugin volume.VolumePlugin) error { mountOptions := util.MountOptionFromSpec(volumeToMount.VolumeSpec) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 3331db86043..7a41fa30208 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -28,6 +28,7 @@ import ( storage "k8s.io/api/storage/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -50,16 +51,23 @@ type cleanupFuncs func() const ( csiNodeLimitUpdateTimeout = 5 * time.Minute csiPodUnschedulableTimeout = 5 * time.Minute + csiResizeWaitPeriod = 5 * time.Minute + // how long to wait for Resizing Condition on PVC to appear + csiResizingConditionWait = 2 * time.Minute ) var _ = utils.SIGDescribe("CSI mock volume", func() { type testParameters struct { - disableAttach bool - attachLimit int - registerDriver bool - podInfo *bool - scName string - nodeSelectorKey string + disableAttach bool + attachLimit int + registerDriver bool + podInfo *bool + scName string + nodeSelectorKey string + enableResizing bool // enable resizing for both CSI mock driver and storageClass. + enableNodeExpansion bool // enable node expansion for CSI mock driver + // just disable resizing on driver it overrides enableResizing flag for CSI mock driver + disableResizingOnDriver bool } type mockDriverSetup struct { @@ -87,8 +95,21 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { } cs := f.ClientSet var err error + driverOpts := drivers.CSIMockDriverOpts{ + RegisterDriver: tp.registerDriver, + PodInfo: tp.podInfo, + AttachLimit: tp.attachLimit, + DisableAttach: tp.disableAttach, + EnableResizing: tp.enableResizing, + EnableNodeExpansion: tp.enableNodeExpansion, + } - m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfo, tp.attachLimit) + // this just disable resizing on driver, keeping resizing on SC enabled. + if tp.disableResizingOnDriver { + driverOpts.EnableResizing = false + } + + m.driver = drivers.InitMockCSIDriver(driverOpts) config, testCleanup := m.driver.PrepareTest(f) m.testCleanups = append(m.testCleanups, testCleanup) m.config = config @@ -127,6 +148,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { if m.tp.scName != "" { scTest.StorageClassName = m.tp.scName } + + if m.tp.enableResizing { + scTest.AllowVolumeExpansion = true + } + nodeSelection := testsuites.NodeSelection{ // The mock driver only works when everything runs on a single node. Name: nodeName, @@ -149,6 +175,23 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { return class, claim, pod } + createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) { + nodeName := m.config.ClientNodeName + nodeSelection := testsuites.NodeSelection{ + Name: nodeName, + } + if len(m.nodeLabel) > 0 { + nodeSelection = testsuites.NodeSelection{ + Selector: m.nodeLabel, + } + } + pod, err := startPausePodWithClaim(m.cs, pvc, nodeSelection, f.Namespace.Name) + if pod != nil { + m.pods = append(m.pods, pod) + } + return pod, err + } + cleanup := func() { cs := f.ClientSet var errs []error @@ -340,6 +383,177 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { }) }) + Context("CSI Volume expansion [Feature:ExpandCSIVolumes]", func() { + tests := []struct { + name string + nodeExpansionRequired bool + disableAttach bool + disableResizingOnDriver bool + expectFailure bool + }{ + { + name: "should expand volume without restarting pod if nodeExpansion=off", + nodeExpansionRequired: false, + }, + { + name: "should expand volume by restarting pod if attach=on, nodeExpansion=on", + nodeExpansionRequired: true, + }, + { + name: "should expand volume by restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + nodeExpansionRequired: true, + }, + { + name: "should not expand volume if resizingOnDriver=off, resizingOnSC=on", + disableResizingOnDriver: true, + expectFailure: true, + }, + } + for _, t := range tests { + test := t + It(t.name, func() { + var err error + tp := testParameters{ + enableResizing: true, + enableNodeExpansion: test.nodeExpansionRequired, + disableResizingOnDriver: test.disableResizingOnDriver, + } + // disabling attach requires drive registration feature + if test.disableAttach { + tp.disableAttach = true + tp.registerDriver = true + } + + init(tp) + defer cleanup() + + ns := f.Namespace.Name + sc, pvc, pod := createPod() + Expect(pod).NotTo(BeNil(), "while creating pod for resizing") + + Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion") + + err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, m.cs) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + if test.expectFailure { + err = waitForResizingCondition(pvc, m.cs, csiResizingConditionWait) + Expect(err).To(HaveOccurred(), "unexpected resizing condition on PVC") + return + } + + By("Waiting for persistent volume resize to finish") + err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + Expect(err).NotTo(HaveOccurred(), "While waiting for CSI PV resize to finish") + + checkPVCSize := func() { + By("Waiting for PVC resize to finish") + pvc, err = waitForFSResize(pvc, m.cs) + Expect(err).NotTo(HaveOccurred(), "while waiting for PVC resize to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + } + + // if node expansion is not required PVC should be resized as well + if !test.nodeExpansionRequired { + checkPVCSize() + } else { + By("Checking for conditions on pvc") + pvc, err = m.cs.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "While fetching pvc after controller resize") + + inProgressConditions := pvc.Status.Conditions + if len(inProgressConditions) > 0 { + Expect(inProgressConditions[0].Type).To(Equal(v1.PersistentVolumeClaimFileSystemResizePending), "pvc must have fs resizing condition") + } + + By("Deleting the previously created pod") + err = framework.DeletePodWithWait(f, m.cs, pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") + + By("Creating a new pod with same volume") + pod2, err := createPodWithPVC(pvc) + Expect(pod2).NotTo(BeNil(), "while creating pod for csi resizing") + Expect(err).NotTo(HaveOccurred(), "while recreating pod for resizing") + + checkPVCSize() + } + }) + } + }) + Context("CSI online volume expansion [Feature:ExpandCSIVolumes][Feature:ExpandInUseVolumes]", func() { + tests := []struct { + name string + disableAttach bool + }{ + { + name: "should expand volume without restarting pod if attach=on, nodeExpansion=on", + }, + { + name: "should expand volume without restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + }, + } + for _, t := range tests { + test := t + It(test.name, func() { + var err error + params := testParameters{enableResizing: true, enableNodeExpansion: true} + if test.disableAttach { + params.disableAttach = true + params.registerDriver = true + } + + init(params) + + defer cleanup() + + sc, pvc, pod := createPod() + Expect(pod).NotTo(BeNil(), "while creating pod for resizing") + + Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion") + + err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, m.cs) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + By("Waiting for persistent volume resize to finish") + err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + Expect(err).NotTo(HaveOccurred(), "While waiting for PV resize to finish") + + By("Waiting for PVC resize to finish") + pvc, err = waitForFSResize(pvc, m.cs) + Expect(err).NotTo(HaveOccurred(), "while waiting for PVC to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + + }) + } + }) + }) func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error { @@ -445,6 +659,49 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t return class, claim, pod } +func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.Pause), + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + ReadOnly: false, + }, + }, + }, + }, + }, + } + + if node.Name != "" { + pod.Spec.NodeName = node.Name + } + if len(node.Selector) != 0 { + pod.Spec.NodeSelector = node.Selector + } + + return cs.CoreV1().Pods(ns).Create(pod) +} + // checkPodInfo tests that NodePublish was called with expected volume_context func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo bool) error { expectedAttributes := map[string]string{ @@ -508,7 +765,7 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai } func waitForCSIDriver(cs clientset.Interface, driverName string) error { - timeout := 2 * time.Minute + timeout := 4 * time.Minute framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName) for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) { diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index fd090adce67..e579913cfd9 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -169,36 +169,52 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per // mockCSI type mockCSIDriver struct { - driverInfo testsuites.DriverInfo - manifests []string - podInfo *bool - attachable bool - attachLimit int + driverInfo testsuites.DriverInfo + manifests []string + podInfo *bool + attachable bool + attachLimit int + enableNodeExpansion bool +} + +// CSIMockDriverOpts defines options used for csi driver +type CSIMockDriverOpts struct { + RegisterDriver bool + DisableAttach bool + PodInfo *bool + AttachLimit int + EnableResizing bool + EnableNodeExpansion bool } var _ testsuites.TestDriver = &mockCSIDriver{} var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface -func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, attachLimit int) testsuites.TestDriver { +func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver { driverManifests := []string{ "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", } - if registerDriver { + if driverOpts.RegisterDriver { driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml") } - if driverAttachable { + if !driverOpts.DisableAttach { driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml") } + if driverOpts.EnableResizing { + driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml") + } + return &mockCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "csi-mock", @@ -213,10 +229,11 @@ func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, att testsuites.CapExec: false, }, }, - manifests: driverManifests, - podInfo: podInfo, - attachable: driverAttachable, - attachLimit: attachLimit, + manifests: driverManifests, + podInfo: driverOpts.PodInfo, + attachable: !driverOpts.DisableAttach, + attachLimit: driverOpts.AttachLimit, + enableNodeExpansion: driverOpts.EnableNodeExpansion, } } @@ -264,6 +281,10 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest containerArgs = append(containerArgs, "--attach-limit", strconv.Itoa(m.attachLimit)) } + if m.enableNodeExpansion { + containerArgs = append(containerArgs, "--node-expand-required=true") + } + // TODO (?): the storage.csi.image.version and storage.csi.image.registry // settings are ignored for this test. We could patch the image definitions. o := utils.PatchCSIOptions{ diff --git a/test/e2e/storage/flexvolume_mounted_volume_resize.go b/test/e2e/storage/flexvolume_mounted_volume_resize.go index 0d95e127fd7..92791c1b784 100644 --- a/test/e2e/storage/flexvolume_mounted_volume_resize.go +++ b/test/e2e/storage/flexvolume_mounted_volume_resize.go @@ -159,7 +159,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume expand[Slow]", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Getting a pod from deployment") diff --git a/test/e2e/storage/flexvolume_online_resize.go b/test/e2e/storage/flexvolume_online_resize.go index f51e078b97c..da88966cf19 100644 --- a/test/e2e/storage/flexvolume_online_resize.go +++ b/test/e2e/storage/flexvolume_online_resize.go @@ -164,7 +164,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume volume expand [Slow] [Feature:Expa } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Waiting for file system resize to finish") diff --git a/test/e2e/storage/mounted_volume_resize.go b/test/e2e/storage/mounted_volume_resize.go index 15f0e041a68..20ba0a64edf 100644 --- a/test/e2e/storage/mounted_volume_resize.go +++ b/test/e2e/storage/mounted_volume_resize.go @@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Mounted volume expand", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Getting a pod from deployment") diff --git a/test/e2e/storage/volume_expand.go b/test/e2e/storage/volume_expand.go index 5d9486d4a94..b67901b6f99 100644 --- a/test/e2e/storage/volume_expand.go +++ b/test/e2e/storage/volume_expand.go @@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Volume expand", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Checking for conditions on pvc") @@ -198,9 +198,29 @@ func expandPVCSize(origPVC *v1.PersistentVolumeClaim, size resource.Quantity, c return updatedPVC, waitErr } -func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) error { +func waitForResizingCondition(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error { + waitErr := wait.PollImmediate(resizePollInterval, duration, func() (bool, error) { + var err error + updatedPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + + if err != nil { + return false, fmt.Errorf("error fetching pvc %q for checking for resize status : %v", pvc.Name, err) + } + + pvcConditions := updatedPVC.Status.Conditions + if len(pvcConditions) > 0 { + if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing { + return true, nil + } + } + return false, nil + }) + return waitErr +} + +func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error { pvName := pvc.Spec.VolumeName - return wait.PollImmediate(resizePollInterval, totalResizeWaitPeriod, func() (bool, error) { + return wait.PollImmediate(resizePollInterval, duration, func() (bool, error) { pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md new file mode 100644 index 00000000000..f7029912efb --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md @@ -0,0 +1 @@ +The original file is (or will be) https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml new file mode 100644 index 00000000000..82b5640b5ed --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml @@ -0,0 +1,90 @@ +# This YAML file contains all RBAC objects that are necessary to run external +# CSI resizer. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# resizer, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-resizer + # replace with non-default namespace name + namespace: default + +--- +# Resizer must be able to work with PVCs, PVs, SCs. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-resizer-runner +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-resizer + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-resizer-runner + apiGroup: rbac.authorization.k8s.io + +--- +# Resizer must be able to work with end point in current namespace +# if (and only if) leadership election is enabled +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: external-resizer-cfg +rules: +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: csi-resizer + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: external-resizer-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml new file mode 100644 index 00000000000..0ea5ce6933b --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml @@ -0,0 +1,33 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-mockplugin-resizer +spec: + selector: + matchLabels: + app: csi-mockplugin-resizer + replicas: 1 + template: + metadata: + labels: + app: csi-mockplugin-resizer + spec: + serviceAccountName: csi-mock + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-mock + type: DirectoryOrCreate + name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml index e3fcb9f44a1..1f1d6494285 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml @@ -54,7 +54,7 @@ spec: - name: mock - image: quay.io/k8scsi/mock-driver:v1.0.0-1 + image: quay.io/k8scsi/mock-driver:v1.1.1 env: - name: CSI_ENDPOINT value: /csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml index dff014b6f20..8936f977d0d 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml @@ -59,3 +59,16 @@ roleRef: kind: ClusterRole name: e2e-test-privileged-psp apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-resizer-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: external-resizer-runner + apiGroup: rbac.authorization.k8s.io diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go index 0652b822f77..d84db2fca17 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -63,7 +63,68 @@ func (x PluginCapability_Service_Type) String() string { return proto.EnumName(PluginCapability_Service_Type_name, int32(x)) } func (PluginCapability_Service_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4, 0, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{4, 0, 0} +} + +type PluginCapability_VolumeExpansion_Type int32 + +const ( + PluginCapability_VolumeExpansion_UNKNOWN PluginCapability_VolumeExpansion_Type = 0 + // ONLINE indicates that volumes may be expanded when published to + // a node. When a Plugin implements this capability it MUST + // implement either the EXPAND_VOLUME controller capability or the + // EXPAND_VOLUME node capability or both. When a plugin supports + // ONLINE volume expansion and also has the EXPAND_VOLUME + // controller capability then the plugin MUST support expansion of + // volumes currently published and available on a node. When a + // plugin supports ONLINE volume expansion and also has the + // EXPAND_VOLUME node capability then the plugin MAY support + // expansion of node-published volume via NodeExpandVolume. + // + // Example 1: Given a shared filesystem volume (e.g. GlusterFs), + // the Plugin may set the ONLINE volume expansion capability and + // implement ControllerExpandVolume but not NodeExpandVolume. + // + // Example 2: Given a block storage volume type (e.g. EBS), the + // Plugin may set the ONLINE volume expansion capability and + // implement both ControllerExpandVolume and NodeExpandVolume. + // + // Example 3: Given a Plugin that supports volume expansion only + // upon a node, the Plugin may set the ONLINE volume + // expansion capability and implement NodeExpandVolume but not + // ControllerExpandVolume. + PluginCapability_VolumeExpansion_ONLINE PluginCapability_VolumeExpansion_Type = 1 + // OFFLINE indicates that volumes currently published and + // available on a node SHALL NOT be expanded via + // ControllerExpandVolume. When a plugin supports OFFLINE volume + // expansion it MUST implement either the EXPAND_VOLUME controller + // capability or both the EXPAND_VOLUME controller capability and + // the EXPAND_VOLUME node capability. + // + // Example 1: Given a block storage volume type (e.g. Azure Disk) + // that does not support expansion of "node-attached" (i.e. + // controller-published) volumes, the Plugin may indicate + // OFFLINE volume expansion support and implement both + // ControllerExpandVolume and NodeExpandVolume. + PluginCapability_VolumeExpansion_OFFLINE PluginCapability_VolumeExpansion_Type = 2 +) + +var PluginCapability_VolumeExpansion_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "ONLINE", + 2: "OFFLINE", +} +var PluginCapability_VolumeExpansion_Type_value = map[string]int32{ + "UNKNOWN": 0, + "ONLINE": 1, + "OFFLINE": 2, +} + +func (x PluginCapability_VolumeExpansion_Type) String() string { + return proto.EnumName(PluginCapability_VolumeExpansion_Type_name, int32(x)) +} +func (PluginCapability_VolumeExpansion_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{4, 1, 0} } type VolumeCapability_AccessMode_Mode int32 @@ -107,7 +168,7 @@ func (x VolumeCapability_AccessMode_Mode) String() string { return proto.EnumName(VolumeCapability_AccessMode_Mode_name, int32(x)) } func (VolumeCapability_AccessMode_Mode) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 2, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{10, 2, 0} } type ControllerServiceCapability_RPC_Type int32 @@ -132,6 +193,8 @@ const ( // Indicates the SP supports ControllerPublishVolume.readonly // field. ControllerServiceCapability_RPC_PUBLISH_READONLY ControllerServiceCapability_RPC_Type = 8 + // See VolumeExpansion for details. + ControllerServiceCapability_RPC_EXPAND_VOLUME ControllerServiceCapability_RPC_Type = 9 ) var ControllerServiceCapability_RPC_Type_name = map[int32]string{ @@ -144,6 +207,7 @@ var ControllerServiceCapability_RPC_Type_name = map[int32]string{ 6: "LIST_SNAPSHOTS", 7: "CLONE_VOLUME", 8: "PUBLISH_READONLY", + 9: "EXPAND_VOLUME", } var ControllerServiceCapability_RPC_Type_value = map[string]int32{ "UNKNOWN": 0, @@ -155,13 +219,14 @@ var ControllerServiceCapability_RPC_Type_value = map[string]int32{ "LIST_SNAPSHOTS": 6, "CLONE_VOLUME": 7, "PUBLISH_READONLY": 8, + "EXPAND_VOLUME": 9, } func (x ControllerServiceCapability_RPC_Type) String() string { return proto.EnumName(ControllerServiceCapability_RPC_Type_name, int32(x)) } func (ControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29, 0, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{29, 0, 0} } type VolumeUsage_Unit int32 @@ -187,7 +252,7 @@ func (x VolumeUsage_Unit) String() string { return proto.EnumName(VolumeUsage_Unit_name, int32(x)) } func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{47, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{49, 0} } type NodeServiceCapability_RPC_Type int32 @@ -199,24 +264,28 @@ const ( // then it MUST implement NodeGetVolumeStats RPC // call for fetching volume statistics. NodeServiceCapability_RPC_GET_VOLUME_STATS NodeServiceCapability_RPC_Type = 2 + // See VolumeExpansion for details. + NodeServiceCapability_RPC_EXPAND_VOLUME NodeServiceCapability_RPC_Type = 3 ) var NodeServiceCapability_RPC_Type_name = map[int32]string{ 0: "UNKNOWN", 1: "STAGE_UNSTAGE_VOLUME", 2: "GET_VOLUME_STATS", + 3: "EXPAND_VOLUME", } var NodeServiceCapability_RPC_Type_value = map[string]int32{ "UNKNOWN": 0, "STAGE_UNSTAGE_VOLUME": 1, "GET_VOLUME_STATS": 2, + "EXPAND_VOLUME": 3, } func (x NodeServiceCapability_RPC_Type) String() string { return proto.EnumName(NodeServiceCapability_RPC_Type_name, int32(x)) } func (NodeServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50, 0, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{52, 0, 0} } type GetPluginInfoRequest struct { @@ -229,7 +298,7 @@ func (m *GetPluginInfoRequest) Reset() { *m = GetPluginInfoRequest{} } func (m *GetPluginInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoRequest) ProtoMessage() {} func (*GetPluginInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{0} + return fileDescriptor_csi_2c5455657a82ae49, []int{0} } func (m *GetPluginInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoRequest.Unmarshal(m, b) @@ -271,7 +340,7 @@ func (m *GetPluginInfoResponse) Reset() { *m = GetPluginInfoResponse{} } func (m *GetPluginInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoResponse) ProtoMessage() {} func (*GetPluginInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{1} + return fileDescriptor_csi_2c5455657a82ae49, []int{1} } func (m *GetPluginInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoResponse.Unmarshal(m, b) @@ -322,7 +391,7 @@ func (m *GetPluginCapabilitiesRequest) Reset() { *m = GetPluginCapabilit func (m *GetPluginCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesRequest) ProtoMessage() {} func (*GetPluginCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{2} + return fileDescriptor_csi_2c5455657a82ae49, []int{2} } func (m *GetPluginCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesRequest.Unmarshal(m, b) @@ -355,7 +424,7 @@ func (m *GetPluginCapabilitiesResponse) Reset() { *m = GetPluginCapabili func (m *GetPluginCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesResponse) ProtoMessage() {} func (*GetPluginCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{3} + return fileDescriptor_csi_2c5455657a82ae49, []int{3} } func (m *GetPluginCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesResponse.Unmarshal(m, b) @@ -386,6 +455,7 @@ func (m *GetPluginCapabilitiesResponse) GetCapabilities() []*PluginCapability { type PluginCapability struct { // Types that are valid to be assigned to Type: // *PluginCapability_Service_ + // *PluginCapability_VolumeExpansion_ Type isPluginCapability_Type `protobuf_oneof:"type"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -396,7 +466,7 @@ func (m *PluginCapability) Reset() { *m = PluginCapability{} } func (m *PluginCapability) String() string { return proto.CompactTextString(m) } func (*PluginCapability) ProtoMessage() {} func (*PluginCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4} + return fileDescriptor_csi_2c5455657a82ae49, []int{4} } func (m *PluginCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability.Unmarshal(m, b) @@ -424,8 +494,14 @@ type PluginCapability_Service_ struct { Service *PluginCapability_Service `protobuf:"bytes,1,opt,name=service,proto3,oneof"` } +type PluginCapability_VolumeExpansion_ struct { + VolumeExpansion *PluginCapability_VolumeExpansion `protobuf:"bytes,2,opt,name=volume_expansion,json=volumeExpansion,proto3,oneof"` +} + func (*PluginCapability_Service_) isPluginCapability_Type() {} +func (*PluginCapability_VolumeExpansion_) isPluginCapability_Type() {} + func (m *PluginCapability) GetType() isPluginCapability_Type { if m != nil { return m.Type @@ -440,10 +516,18 @@ func (m *PluginCapability) GetService() *PluginCapability_Service { return nil } +func (m *PluginCapability) GetVolumeExpansion() *PluginCapability_VolumeExpansion { + if x, ok := m.GetType().(*PluginCapability_VolumeExpansion_); ok { + return x.VolumeExpansion + } + return nil +} + // XXX_OneofFuncs is for the internal use of the proto package. func (*PluginCapability) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { return _PluginCapability_OneofMarshaler, _PluginCapability_OneofUnmarshaler, _PluginCapability_OneofSizer, []interface{}{ (*PluginCapability_Service_)(nil), + (*PluginCapability_VolumeExpansion_)(nil), } } @@ -456,6 +540,11 @@ func _PluginCapability_OneofMarshaler(msg proto.Message, b *proto.Buffer) error if err := b.EncodeMessage(x.Service); err != nil { return err } + case *PluginCapability_VolumeExpansion_: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.VolumeExpansion); err != nil { + return err + } case nil: default: return fmt.Errorf("PluginCapability.Type has unexpected type %T", x) @@ -474,6 +563,14 @@ func _PluginCapability_OneofUnmarshaler(msg proto.Message, tag, wire int, b *pro err := b.DecodeMessage(msg) m.Type = &PluginCapability_Service_{msg} return true, err + case 2: // type.volume_expansion + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(PluginCapability_VolumeExpansion) + err := b.DecodeMessage(msg) + m.Type = &PluginCapability_VolumeExpansion_{msg} + return true, err default: return false, nil } @@ -488,6 +585,11 @@ func _PluginCapability_OneofSizer(msg proto.Message) (n int) { n += 1 // tag and wire n += proto.SizeVarint(uint64(s)) n += s + case *PluginCapability_VolumeExpansion_: + s := proto.Size(x.VolumeExpansion) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s case nil: default: panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) @@ -506,7 +608,7 @@ func (m *PluginCapability_Service) Reset() { *m = PluginCapability_Servi func (m *PluginCapability_Service) String() string { return proto.CompactTextString(m) } func (*PluginCapability_Service) ProtoMessage() {} func (*PluginCapability_Service) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{4, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{4, 0} } func (m *PluginCapability_Service) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability_Service.Unmarshal(m, b) @@ -533,6 +635,44 @@ func (m *PluginCapability_Service) GetType() PluginCapability_Service_Type { return PluginCapability_Service_UNKNOWN } +type PluginCapability_VolumeExpansion struct { + Type PluginCapability_VolumeExpansion_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.PluginCapability_VolumeExpansion_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PluginCapability_VolumeExpansion) Reset() { *m = PluginCapability_VolumeExpansion{} } +func (m *PluginCapability_VolumeExpansion) String() string { return proto.CompactTextString(m) } +func (*PluginCapability_VolumeExpansion) ProtoMessage() {} +func (*PluginCapability_VolumeExpansion) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{4, 1} +} +func (m *PluginCapability_VolumeExpansion) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Unmarshal(m, b) +} +func (m *PluginCapability_VolumeExpansion) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Marshal(b, m, deterministic) +} +func (dst *PluginCapability_VolumeExpansion) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability_VolumeExpansion.Merge(dst, src) +} +func (m *PluginCapability_VolumeExpansion) XXX_Size() int { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Size(m) +} +func (m *PluginCapability_VolumeExpansion) XXX_DiscardUnknown() { + xxx_messageInfo_PluginCapability_VolumeExpansion.DiscardUnknown(m) +} + +var xxx_messageInfo_PluginCapability_VolumeExpansion proto.InternalMessageInfo + +func (m *PluginCapability_VolumeExpansion) GetType() PluginCapability_VolumeExpansion_Type { + if m != nil { + return m.Type + } + return PluginCapability_VolumeExpansion_UNKNOWN +} + type ProbeRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -543,7 +683,7 @@ func (m *ProbeRequest) Reset() { *m = ProbeRequest{} } func (m *ProbeRequest) String() string { return proto.CompactTextString(m) } func (*ProbeRequest) ProtoMessage() {} func (*ProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{5} + return fileDescriptor_csi_2c5455657a82ae49, []int{5} } func (m *ProbeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeRequest.Unmarshal(m, b) @@ -594,7 +734,7 @@ func (m *ProbeResponse) Reset() { *m = ProbeResponse{} } func (m *ProbeResponse) String() string { return proto.CompactTextString(m) } func (*ProbeResponse) ProtoMessage() {} func (*ProbeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{6} + return fileDescriptor_csi_2c5455657a82ae49, []int{6} } func (m *ProbeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeResponse.Unmarshal(m, b) @@ -703,7 +843,7 @@ func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } func (*CreateVolumeRequest) ProtoMessage() {} func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{7} + return fileDescriptor_csi_2c5455657a82ae49, []int{7} } func (m *CreateVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeRequest.Unmarshal(m, b) @@ -788,7 +928,7 @@ func (m *VolumeContentSource) Reset() { *m = VolumeContentSource{} } func (m *VolumeContentSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource) ProtoMessage() {} func (*VolumeContentSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8} + return fileDescriptor_csi_2c5455657a82ae49, []int{8} } func (m *VolumeContentSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource.Unmarshal(m, b) @@ -934,7 +1074,7 @@ func (m *VolumeContentSource_SnapshotSource) Reset() { *m = VolumeConten func (m *VolumeContentSource_SnapshotSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource_SnapshotSource) ProtoMessage() {} func (*VolumeContentSource_SnapshotSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{8, 0} } func (m *VolumeContentSource_SnapshotSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource_SnapshotSource.Unmarshal(m, b) @@ -975,7 +1115,7 @@ func (m *VolumeContentSource_VolumeSource) Reset() { *m = VolumeContentS func (m *VolumeContentSource_VolumeSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource_VolumeSource) ProtoMessage() {} func (*VolumeContentSource_VolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{8, 1} + return fileDescriptor_csi_2c5455657a82ae49, []int{8, 1} } func (m *VolumeContentSource_VolumeSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource_VolumeSource.Unmarshal(m, b) @@ -1016,7 +1156,7 @@ func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } func (*CreateVolumeResponse) ProtoMessage() {} func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{9} + return fileDescriptor_csi_2c5455657a82ae49, []int{9} } func (m *CreateVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeResponse.Unmarshal(m, b) @@ -1063,7 +1203,7 @@ func (m *VolumeCapability) Reset() { *m = VolumeCapability{} } func (m *VolumeCapability) String() string { return proto.CompactTextString(m) } func (*VolumeCapability) ProtoMessage() {} func (*VolumeCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10} + return fileDescriptor_csi_2c5455657a82ae49, []int{10} } func (m *VolumeCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability.Unmarshal(m, b) @@ -1212,7 +1352,7 @@ func (m *VolumeCapability_BlockVolume) Reset() { *m = VolumeCapability_B func (m *VolumeCapability_BlockVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_BlockVolume) ProtoMessage() {} func (*VolumeCapability_BlockVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{10, 0} } func (m *VolumeCapability_BlockVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_BlockVolume.Unmarshal(m, b) @@ -1252,7 +1392,7 @@ func (m *VolumeCapability_MountVolume) Reset() { *m = VolumeCapability_M func (m *VolumeCapability_MountVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_MountVolume) ProtoMessage() {} func (*VolumeCapability_MountVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 1} + return fileDescriptor_csi_2c5455657a82ae49, []int{10, 1} } func (m *VolumeCapability_MountVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_MountVolume.Unmarshal(m, b) @@ -1299,7 +1439,7 @@ func (m *VolumeCapability_AccessMode) Reset() { *m = VolumeCapability_Ac func (m *VolumeCapability_AccessMode) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_AccessMode) ProtoMessage() {} func (*VolumeCapability_AccessMode) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{10, 2} + return fileDescriptor_csi_2c5455657a82ae49, []int{10, 2} } func (m *VolumeCapability_AccessMode) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_AccessMode.Unmarshal(m, b) @@ -1347,7 +1487,7 @@ func (m *CapacityRange) Reset() { *m = CapacityRange{} } func (m *CapacityRange) String() string { return proto.CompactTextString(m) } func (*CapacityRange) ProtoMessage() {} func (*CapacityRange) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{11} + return fileDescriptor_csi_2c5455657a82ae49, []int{11} } func (m *CapacityRange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapacityRange.Unmarshal(m, b) @@ -1452,7 +1592,7 @@ func (m *Volume) Reset() { *m = Volume{} } func (m *Volume) String() string { return proto.CompactTextString(m) } func (*Volume) ProtoMessage() {} func (*Volume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{12} + return fileDescriptor_csi_2c5455657a82ae49, []int{12} } func (m *Volume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Volume.Unmarshal(m, b) @@ -1520,7 +1660,7 @@ type TopologyRequirement struct { // x = number of topologies provisioned volume is accessible from // n = number of requisite topologies // The CO MUST ensure n >= 1. The SP MUST ensure x >= 1 - // If x==n, than the SP MUST make the provisioned volume available to + // If x==n, then the SP MUST make the provisioned volume available to // all topologies from the list of requisite topologies. If it is // unable to do so, the SP MUST fail the CreateVolume call. // For example, if a volume should be accessible from a single zone, @@ -1535,7 +1675,7 @@ type TopologyRequirement struct { // then the provisioned volume MUST be accessible from the "region" // "R1" and both "zone" "Z2" and "zone" "Z3". // - // If xn, than the SP MUST make the provisioned volume available from + // If x>n, then the SP MUST make the provisioned volume available from // all topologies from the list of requisite topologies and MAY choose // the remaining x-n unique topologies from the list of all possible // topologies. If it is unable to do so, the SP MUST fail the @@ -1644,7 +1784,7 @@ func (m *TopologyRequirement) Reset() { *m = TopologyRequirement{} } func (m *TopologyRequirement) String() string { return proto.CompactTextString(m) } func (*TopologyRequirement) ProtoMessage() {} func (*TopologyRequirement) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{13} + return fileDescriptor_csi_2c5455657a82ae49, []int{13} } func (m *TopologyRequirement) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TopologyRequirement.Unmarshal(m, b) @@ -1718,7 +1858,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{14} + return fileDescriptor_csi_2c5455657a82ae49, []int{14} } func (m *Topology) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Topology.Unmarshal(m, b) @@ -1762,7 +1902,7 @@ func (m *DeleteVolumeRequest) Reset() { *m = DeleteVolumeRequest{} } func (m *DeleteVolumeRequest) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeRequest) ProtoMessage() {} func (*DeleteVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{15} + return fileDescriptor_csi_2c5455657a82ae49, []int{15} } func (m *DeleteVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeRequest.Unmarshal(m, b) @@ -1806,7 +1946,7 @@ func (m *DeleteVolumeResponse) Reset() { *m = DeleteVolumeResponse{} } func (m *DeleteVolumeResponse) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeResponse) ProtoMessage() {} func (*DeleteVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{16} + return fileDescriptor_csi_2c5455657a82ae49, []int{16} } func (m *DeleteVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeResponse.Unmarshal(m, b) @@ -1860,7 +2000,7 @@ func (m *ControllerPublishVolumeRequest) Reset() { *m = ControllerPublis func (m *ControllerPublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeRequest) ProtoMessage() {} func (*ControllerPublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{17} + return fileDescriptor_csi_2c5455657a82ae49, []int{17} } func (m *ControllerPublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeRequest.Unmarshal(m, b) @@ -1946,7 +2086,7 @@ func (m *ControllerPublishVolumeResponse) Reset() { *m = ControllerPubli func (m *ControllerPublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeResponse) ProtoMessage() {} func (*ControllerPublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{18} + return fileDescriptor_csi_2c5455657a82ae49, []int{18} } func (m *ControllerPublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeResponse.Unmarshal(m, b) @@ -1997,7 +2137,7 @@ func (m *ControllerUnpublishVolumeRequest) Reset() { *m = ControllerUnpu func (m *ControllerUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeRequest) ProtoMessage() {} func (*ControllerUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{19} + return fileDescriptor_csi_2c5455657a82ae49, []int{19} } func (m *ControllerUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeRequest.Unmarshal(m, b) @@ -2048,7 +2188,7 @@ func (m *ControllerUnpublishVolumeResponse) Reset() { *m = ControllerUnp func (m *ControllerUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeResponse) ProtoMessage() {} func (*ControllerUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{20} + return fileDescriptor_csi_2c5455657a82ae49, []int{20} } func (m *ControllerUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeResponse.Unmarshal(m, b) @@ -2095,7 +2235,7 @@ func (m *ValidateVolumeCapabilitiesRequest) Reset() { *m = ValidateVolum func (m *ValidateVolumeCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesRequest) ProtoMessage() {} func (*ValidateVolumeCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{21} + return fileDescriptor_csi_2c5455657a82ae49, []int{21} } func (m *ValidateVolumeCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Unmarshal(m, b) @@ -2173,7 +2313,7 @@ func (m *ValidateVolumeCapabilitiesResponse) Reset() { *m = ValidateVolu func (m *ValidateVolumeCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesResponse) ProtoMessage() {} func (*ValidateVolumeCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{22} + return fileDescriptor_csi_2c5455657a82ae49, []int{22} } func (m *ValidateVolumeCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Unmarshal(m, b) @@ -2230,7 +2370,7 @@ func (m *ValidateVolumeCapabilitiesResponse_Confirmed) String() string { } func (*ValidateVolumeCapabilitiesResponse_Confirmed) ProtoMessage() {} func (*ValidateVolumeCapabilitiesResponse_Confirmed) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{22, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{22, 0} } func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Unmarshal(m, b) @@ -2295,7 +2435,7 @@ func (m *ListVolumesRequest) Reset() { *m = ListVolumesRequest{} } func (m *ListVolumesRequest) String() string { return proto.CompactTextString(m) } func (*ListVolumesRequest) ProtoMessage() {} func (*ListVolumesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{23} + return fileDescriptor_csi_2c5455657a82ae49, []int{23} } func (m *ListVolumesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesRequest.Unmarshal(m, b) @@ -2347,7 +2487,7 @@ func (m *ListVolumesResponse) Reset() { *m = ListVolumesResponse{} } func (m *ListVolumesResponse) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse) ProtoMessage() {} func (*ListVolumesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{24} + return fileDescriptor_csi_2c5455657a82ae49, []int{24} } func (m *ListVolumesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse.Unmarshal(m, b) @@ -2392,7 +2532,7 @@ func (m *ListVolumesResponse_Entry) Reset() { *m = ListVolumesResponse_E func (m *ListVolumesResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse_Entry) ProtoMessage() {} func (*ListVolumesResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{24, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{24, 0} } func (m *ListVolumesResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse_Entry.Unmarshal(m, b) @@ -2447,7 +2587,7 @@ func (m *GetCapacityRequest) Reset() { *m = GetCapacityRequest{} } func (m *GetCapacityRequest) String() string { return proto.CompactTextString(m) } func (*GetCapacityRequest) ProtoMessage() {} func (*GetCapacityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{25} + return fileDescriptor_csi_2c5455657a82ae49, []int{25} } func (m *GetCapacityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityRequest.Unmarshal(m, b) @@ -2505,7 +2645,7 @@ func (m *GetCapacityResponse) Reset() { *m = GetCapacityResponse{} } func (m *GetCapacityResponse) String() string { return proto.CompactTextString(m) } func (*GetCapacityResponse) ProtoMessage() {} func (*GetCapacityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{26} + return fileDescriptor_csi_2c5455657a82ae49, []int{26} } func (m *GetCapacityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityResponse.Unmarshal(m, b) @@ -2542,7 +2682,7 @@ func (m *ControllerGetCapabilitiesRequest) Reset() { *m = ControllerGetC func (m *ControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesRequest) ProtoMessage() {} func (*ControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{27} + return fileDescriptor_csi_2c5455657a82ae49, []int{27} } func (m *ControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesRequest.Unmarshal(m, b) @@ -2575,7 +2715,7 @@ func (m *ControllerGetCapabilitiesResponse) Reset() { *m = ControllerGet func (m *ControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesResponse) ProtoMessage() {} func (*ControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{28} + return fileDescriptor_csi_2c5455657a82ae49, []int{28} } func (m *ControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesResponse.Unmarshal(m, b) @@ -2616,7 +2756,7 @@ func (m *ControllerServiceCapability) Reset() { *m = ControllerServiceCa func (m *ControllerServiceCapability) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability) ProtoMessage() {} func (*ControllerServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29} + return fileDescriptor_csi_2c5455657a82ae49, []int{29} } func (m *ControllerServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability.Unmarshal(m, b) @@ -2726,7 +2866,7 @@ func (m *ControllerServiceCapability_RPC) Reset() { *m = ControllerServi func (m *ControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability_RPC) ProtoMessage() {} func (*ControllerServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{29, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{29, 0} } func (m *ControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability_RPC.Unmarshal(m, b) @@ -2788,7 +2928,7 @@ func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotRequest) ProtoMessage() {} func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{30} + return fileDescriptor_csi_2c5455657a82ae49, []int{30} } func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) @@ -2850,7 +2990,7 @@ func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotResponse) ProtoMessage() {} func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{31} + return fileDescriptor_csi_2c5455657a82ae49, []int{31} } func (m *CreateSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotResponse.Unmarshal(m, b) @@ -2917,7 +3057,7 @@ func (m *Snapshot) Reset() { *m = Snapshot{} } func (m *Snapshot) String() string { return proto.CompactTextString(m) } func (*Snapshot) ProtoMessage() {} func (*Snapshot) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{32} + return fileDescriptor_csi_2c5455657a82ae49, []int{32} } func (m *Snapshot) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Snapshot.Unmarshal(m, b) @@ -2989,7 +3129,7 @@ func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotRequest) ProtoMessage() {} func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{33} + return fileDescriptor_csi_2c5455657a82ae49, []int{33} } func (m *DeleteSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotRequest.Unmarshal(m, b) @@ -3033,7 +3173,7 @@ func (m *DeleteSnapshotResponse) Reset() { *m = DeleteSnapshotResponse{} func (m *DeleteSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotResponse) ProtoMessage() {} func (*DeleteSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{34} + return fileDescriptor_csi_2c5455657a82ae49, []int{34} } func (m *DeleteSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotResponse.Unmarshal(m, b) @@ -3088,7 +3228,7 @@ func (m *ListSnapshotsRequest) Reset() { *m = ListSnapshotsRequest{} } func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsRequest) ProtoMessage() {} func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{35} + return fileDescriptor_csi_2c5455657a82ae49, []int{35} } func (m *ListSnapshotsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsRequest.Unmarshal(m, b) @@ -3154,7 +3294,7 @@ func (m *ListSnapshotsResponse) Reset() { *m = ListSnapshotsResponse{} } func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse) ProtoMessage() {} func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{36} + return fileDescriptor_csi_2c5455657a82ae49, []int{36} } func (m *ListSnapshotsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse.Unmarshal(m, b) @@ -3199,7 +3339,7 @@ func (m *ListSnapshotsResponse_Entry) Reset() { *m = ListSnapshotsRespon func (m *ListSnapshotsResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse_Entry) ProtoMessage() {} func (*ListSnapshotsResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{36, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{36, 0} } func (m *ListSnapshotsResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse_Entry.Unmarshal(m, b) @@ -3226,6 +3366,115 @@ func (m *ListSnapshotsResponse_Entry) GetSnapshot() *Snapshot { return nil } +type ControllerExpandVolumeRequest struct { + // The ID of the volume to expand. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // This allows CO to specify the capacity requirements of the volume + // after expansion. This field is REQUIRED. + CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // Secrets required by the plugin for expanding the volume. + // This field is OPTIONAL. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerExpandVolumeRequest) Reset() { *m = ControllerExpandVolumeRequest{} } +func (m *ControllerExpandVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerExpandVolumeRequest) ProtoMessage() {} +func (*ControllerExpandVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{37} +} +func (m *ControllerExpandVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerExpandVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerExpandVolumeRequest.Marshal(b, m, deterministic) +} +func (dst *ControllerExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeRequest.Merge(dst, src) +} +func (m *ControllerExpandVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerExpandVolumeRequest.Size(m) +} +func (m *ControllerExpandVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerExpandVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerExpandVolumeRequest proto.InternalMessageInfo + +func (m *ControllerExpandVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ControllerExpandVolumeRequest) GetCapacityRange() *CapacityRange { + if m != nil { + return m.CapacityRange + } + return nil +} + +func (m *ControllerExpandVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type ControllerExpandVolumeResponse struct { + // Capacity of volume after expansion. This field is REQUIRED. + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + // Whether node expansion is required for the volume. When true + // the CO MUST make NodeExpandVolume RPC call on the node. This field + // is REQUIRED. + NodeExpansionRequired bool `protobuf:"varint,2,opt,name=node_expansion_required,json=nodeExpansionRequired,proto3" json:"node_expansion_required,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerExpandVolumeResponse) Reset() { *m = ControllerExpandVolumeResponse{} } +func (m *ControllerExpandVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerExpandVolumeResponse) ProtoMessage() {} +func (*ControllerExpandVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{38} +} +func (m *ControllerExpandVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerExpandVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerExpandVolumeResponse.Marshal(b, m, deterministic) +} +func (dst *ControllerExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeResponse.Merge(dst, src) +} +func (m *ControllerExpandVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerExpandVolumeResponse.Size(m) +} +func (m *ControllerExpandVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerExpandVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerExpandVolumeResponse proto.InternalMessageInfo + +func (m *ControllerExpandVolumeResponse) GetCapacityBytes() int64 { + if m != nil { + return m.CapacityBytes + } + return 0 +} + +func (m *ControllerExpandVolumeResponse) GetNodeExpansionRequired() bool { + if m != nil { + return m.NodeExpansionRequired + } + return false +} + type NodeStageVolumeRequest struct { // The ID of the volume to publish. This field is REQUIRED. VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` @@ -3267,7 +3516,7 @@ func (m *NodeStageVolumeRequest) Reset() { *m = NodeStageVolumeRequest{} func (m *NodeStageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeRequest) ProtoMessage() {} func (*NodeStageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{37} + return fileDescriptor_csi_2c5455657a82ae49, []int{39} } func (m *NodeStageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeRequest.Unmarshal(m, b) @@ -3339,7 +3588,7 @@ func (m *NodeStageVolumeResponse) Reset() { *m = NodeStageVolumeResponse func (m *NodeStageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeResponse) ProtoMessage() {} func (*NodeStageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{38} + return fileDescriptor_csi_2c5455657a82ae49, []int{40} } func (m *NodeStageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeResponse.Unmarshal(m, b) @@ -3375,7 +3624,7 @@ func (m *NodeUnstageVolumeRequest) Reset() { *m = NodeUnstageVolumeReque func (m *NodeUnstageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeRequest) ProtoMessage() {} func (*NodeUnstageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{39} + return fileDescriptor_csi_2c5455657a82ae49, []int{41} } func (m *NodeUnstageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeRequest.Unmarshal(m, b) @@ -3419,7 +3668,7 @@ func (m *NodeUnstageVolumeResponse) Reset() { *m = NodeUnstageVolumeResp func (m *NodeUnstageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeResponse) ProtoMessage() {} func (*NodeUnstageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{40} + return fileDescriptor_csi_2c5455657a82ae49, []int{42} } func (m *NodeUnstageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeResponse.Unmarshal(m, b) @@ -3493,7 +3742,7 @@ func (m *NodePublishVolumeRequest) Reset() { *m = NodePublishVolumeReque func (m *NodePublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeRequest) ProtoMessage() {} func (*NodePublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{41} + return fileDescriptor_csi_2c5455657a82ae49, []int{43} } func (m *NodePublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeRequest.Unmarshal(m, b) @@ -3579,7 +3828,7 @@ func (m *NodePublishVolumeResponse) Reset() { *m = NodePublishVolumeResp func (m *NodePublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeResponse) ProtoMessage() {} func (*NodePublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{42} + return fileDescriptor_csi_2c5455657a82ae49, []int{44} } func (m *NodePublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeResponse.Unmarshal(m, b) @@ -3616,7 +3865,7 @@ func (m *NodeUnpublishVolumeRequest) Reset() { *m = NodeUnpublishVolumeR func (m *NodeUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeRequest) ProtoMessage() {} func (*NodeUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{43} + return fileDescriptor_csi_2c5455657a82ae49, []int{45} } func (m *NodeUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeRequest.Unmarshal(m, b) @@ -3660,7 +3909,7 @@ func (m *NodeUnpublishVolumeResponse) Reset() { *m = NodeUnpublishVolume func (m *NodeUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeResponse) ProtoMessage() {} func (*NodeUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{44} + return fileDescriptor_csi_2c5455657a82ae49, []int{46} } func (m *NodeUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeResponse.Unmarshal(m, b) @@ -3698,7 +3947,7 @@ func (m *NodeGetVolumeStatsRequest) Reset() { *m = NodeGetVolumeStatsReq func (m *NodeGetVolumeStatsRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetVolumeStatsRequest) ProtoMessage() {} func (*NodeGetVolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{45} + return fileDescriptor_csi_2c5455657a82ae49, []int{47} } func (m *NodeGetVolumeStatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetVolumeStatsRequest.Unmarshal(m, b) @@ -3744,7 +3993,7 @@ func (m *NodeGetVolumeStatsResponse) Reset() { *m = NodeGetVolumeStatsRe func (m *NodeGetVolumeStatsResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetVolumeStatsResponse) ProtoMessage() {} func (*NodeGetVolumeStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{46} + return fileDescriptor_csi_2c5455657a82ae49, []int{48} } func (m *NodeGetVolumeStatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetVolumeStatsResponse.Unmarshal(m, b) @@ -3792,7 +4041,7 @@ func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } func (m *VolumeUsage) String() string { return proto.CompactTextString(m) } func (*VolumeUsage) ProtoMessage() {} func (*VolumeUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{47} + return fileDescriptor_csi_2c5455657a82ae49, []int{49} } func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeUsage.Unmarshal(m, b) @@ -3850,7 +4099,7 @@ func (m *NodeGetCapabilitiesRequest) Reset() { *m = NodeGetCapabilitiesR func (m *NodeGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesRequest) ProtoMessage() {} func (*NodeGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{48} + return fileDescriptor_csi_2c5455657a82ae49, []int{50} } func (m *NodeGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesRequest.Unmarshal(m, b) @@ -3883,7 +4132,7 @@ func (m *NodeGetCapabilitiesResponse) Reset() { *m = NodeGetCapabilities func (m *NodeGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesResponse) ProtoMessage() {} func (*NodeGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{49} + return fileDescriptor_csi_2c5455657a82ae49, []int{51} } func (m *NodeGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesResponse.Unmarshal(m, b) @@ -3924,7 +4173,7 @@ func (m *NodeServiceCapability) Reset() { *m = NodeServiceCapability{} } func (m *NodeServiceCapability) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability) ProtoMessage() {} func (*NodeServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50} + return fileDescriptor_csi_2c5455657a82ae49, []int{52} } func (m *NodeServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability.Unmarshal(m, b) @@ -4034,7 +4283,7 @@ func (m *NodeServiceCapability_RPC) Reset() { *m = NodeServiceCapability func (m *NodeServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability_RPC) ProtoMessage() {} func (*NodeServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{50, 0} + return fileDescriptor_csi_2c5455657a82ae49, []int{52, 0} } func (m *NodeServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability_RPC.Unmarshal(m, b) @@ -4071,7 +4320,7 @@ func (m *NodeGetInfoRequest) Reset() { *m = NodeGetInfoRequest{} } func (m *NodeGetInfoRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoRequest) ProtoMessage() {} func (*NodeGetInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{51} + return fileDescriptor_csi_2c5455657a82ae49, []int{53} } func (m *NodeGetInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoRequest.Unmarshal(m, b) @@ -4121,7 +4370,7 @@ type NodeGetInfoResponse struct { // // Example 1: // accessible_topology = - // {"region": "R1", "zone": "R2"} + // {"region": "R1", "zone": "Z2"} // Indicates the node exists within the "region" "R1" and the "zone" // "Z2". AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` @@ -4134,7 +4383,7 @@ func (m *NodeGetInfoResponse) Reset() { *m = NodeGetInfoResponse{} } func (m *NodeGetInfoResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoResponse) ProtoMessage() {} func (*NodeGetInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_1092d4f3f3c8dc30, []int{52} + return fileDescriptor_csi_2c5455657a82ae49, []int{54} } func (m *NodeGetInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoResponse.Unmarshal(m, b) @@ -4175,6 +4424,107 @@ func (m *NodeGetInfoResponse) GetAccessibleTopology() *Topology { return nil } +type NodeExpandVolumeRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The path on which volume is available. This field is REQUIRED. + VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` + // This allows CO to specify the capacity requirements of the volume + // after expansion. If capacity_range is omitted then a plugin MAY + // inspect the file system of the volume to determine the maximum + // capacity to which the volume can be expanded. In such cases a + // plugin MAY expand the volume to its maximum capacity. + // This field is OPTIONAL. + CapacityRange *CapacityRange `protobuf:"bytes,3,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeExpandVolumeRequest) Reset() { *m = NodeExpandVolumeRequest{} } +func (m *NodeExpandVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodeExpandVolumeRequest) ProtoMessage() {} +func (*NodeExpandVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{55} +} +func (m *NodeExpandVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeExpandVolumeRequest.Unmarshal(m, b) +} +func (m *NodeExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeExpandVolumeRequest.Marshal(b, m, deterministic) +} +func (dst *NodeExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeRequest.Merge(dst, src) +} +func (m *NodeExpandVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodeExpandVolumeRequest.Size(m) +} +func (m *NodeExpandVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeExpandVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeExpandVolumeRequest proto.InternalMessageInfo + +func (m *NodeExpandVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetVolumePath() string { + if m != nil { + return m.VolumePath + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetCapacityRange() *CapacityRange { + if m != nil { + return m.CapacityRange + } + return nil +} + +type NodeExpandVolumeResponse struct { + // The capacity of the volume in bytes. This field is OPTIONAL. + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeExpandVolumeResponse) Reset() { *m = NodeExpandVolumeResponse{} } +func (m *NodeExpandVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodeExpandVolumeResponse) ProtoMessage() {} +func (*NodeExpandVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_csi_2c5455657a82ae49, []int{56} +} +func (m *NodeExpandVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeExpandVolumeResponse.Unmarshal(m, b) +} +func (m *NodeExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeExpandVolumeResponse.Marshal(b, m, deterministic) +} +func (dst *NodeExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeResponse.Merge(dst, src) +} +func (m *NodeExpandVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodeExpandVolumeResponse.Size(m) +} +func (m *NodeExpandVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeExpandVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeExpandVolumeResponse proto.InternalMessageInfo + +func (m *NodeExpandVolumeResponse) GetCapacityBytes() int64 { + if m != nil { + return m.CapacityBytes + } + return 0 +} + var E_CsiSecret = &proto.ExtensionDesc{ ExtendedType: (*descriptor.FieldOptions)(nil), ExtensionType: (*bool)(nil), @@ -4192,6 +4542,7 @@ func init() { proto.RegisterType((*GetPluginCapabilitiesResponse)(nil), "csi.v1.GetPluginCapabilitiesResponse") proto.RegisterType((*PluginCapability)(nil), "csi.v1.PluginCapability") proto.RegisterType((*PluginCapability_Service)(nil), "csi.v1.PluginCapability.Service") + proto.RegisterType((*PluginCapability_VolumeExpansion)(nil), "csi.v1.PluginCapability.VolumeExpansion") proto.RegisterType((*ProbeRequest)(nil), "csi.v1.ProbeRequest") proto.RegisterType((*ProbeResponse)(nil), "csi.v1.ProbeResponse") proto.RegisterType((*CreateVolumeRequest)(nil), "csi.v1.CreateVolumeRequest") @@ -4251,6 +4602,9 @@ func init() { proto.RegisterType((*ListSnapshotsRequest)(nil), "csi.v1.ListSnapshotsRequest") proto.RegisterType((*ListSnapshotsResponse)(nil), "csi.v1.ListSnapshotsResponse") proto.RegisterType((*ListSnapshotsResponse_Entry)(nil), "csi.v1.ListSnapshotsResponse.Entry") + proto.RegisterType((*ControllerExpandVolumeRequest)(nil), "csi.v1.ControllerExpandVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerExpandVolumeRequest.SecretsEntry") + proto.RegisterType((*ControllerExpandVolumeResponse)(nil), "csi.v1.ControllerExpandVolumeResponse") proto.RegisterType((*NodeStageVolumeRequest)(nil), "csi.v1.NodeStageVolumeRequest") proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.PublishContextEntry") proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.SecretsEntry") @@ -4274,7 +4628,10 @@ func init() { proto.RegisterType((*NodeServiceCapability_RPC)(nil), "csi.v1.NodeServiceCapability.RPC") proto.RegisterType((*NodeGetInfoRequest)(nil), "csi.v1.NodeGetInfoRequest") proto.RegisterType((*NodeGetInfoResponse)(nil), "csi.v1.NodeGetInfoResponse") + proto.RegisterType((*NodeExpandVolumeRequest)(nil), "csi.v1.NodeExpandVolumeRequest") + proto.RegisterType((*NodeExpandVolumeResponse)(nil), "csi.v1.NodeExpandVolumeResponse") proto.RegisterEnum("csi.v1.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) + proto.RegisterEnum("csi.v1.PluginCapability_VolumeExpansion_Type", PluginCapability_VolumeExpansion_Type_name, PluginCapability_VolumeExpansion_Type_value) proto.RegisterEnum("csi.v1.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) @@ -4435,6 +4792,7 @@ type ControllerClient interface { CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*DeleteSnapshotResponse, error) ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) + ControllerExpandVolume(ctx context.Context, in *ControllerExpandVolumeRequest, opts ...grpc.CallOption) (*ControllerExpandVolumeResponse, error) } type controllerClient struct { @@ -4544,6 +4902,15 @@ func (c *controllerClient) ListSnapshots(ctx context.Context, in *ListSnapshotsR return out, nil } +func (c *controllerClient) ControllerExpandVolume(ctx context.Context, in *ControllerExpandVolumeRequest, opts ...grpc.CallOption) (*ControllerExpandVolumeResponse, error) { + out := new(ControllerExpandVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerExpandVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControllerServer is the server API for Controller service. type ControllerServer interface { CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) @@ -4557,6 +4924,7 @@ type ControllerServer interface { CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotResponse, error) DeleteSnapshot(context.Context, *DeleteSnapshotRequest) (*DeleteSnapshotResponse, error) ListSnapshots(context.Context, *ListSnapshotsRequest) (*ListSnapshotsResponse, error) + ControllerExpandVolume(context.Context, *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error) } func RegisterControllerServer(s *grpc.Server, srv ControllerServer) { @@ -4761,6 +5129,24 @@ func _Controller_ListSnapshots_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Controller_ControllerExpandVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerExpandVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerExpandVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerExpandVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerExpandVolume(ctx, req.(*ControllerExpandVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _Controller_serviceDesc = grpc.ServiceDesc{ ServiceName: "csi.v1.Controller", HandlerType: (*ControllerServer)(nil), @@ -4809,6 +5195,10 @@ var _Controller_serviceDesc = grpc.ServiceDesc{ MethodName: "ListSnapshots", Handler: _Controller_ListSnapshots_Handler, }, + { + MethodName: "ControllerExpandVolume", + Handler: _Controller_ControllerExpandVolume_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "github.com/container-storage-interface/spec/csi.proto", @@ -4823,6 +5213,7 @@ type NodeClient interface { NodePublishVolume(ctx context.Context, in *NodePublishVolumeRequest, opts ...grpc.CallOption) (*NodePublishVolumeResponse, error) NodeUnpublishVolume(ctx context.Context, in *NodeUnpublishVolumeRequest, opts ...grpc.CallOption) (*NodeUnpublishVolumeResponse, error) NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*NodeGetVolumeStatsResponse, error) + NodeExpandVolume(ctx context.Context, in *NodeExpandVolumeRequest, opts ...grpc.CallOption) (*NodeExpandVolumeResponse, error) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) NodeGetInfo(ctx context.Context, in *NodeGetInfoRequest, opts ...grpc.CallOption) (*NodeGetInfoResponse, error) } @@ -4880,6 +5271,15 @@ func (c *nodeClient) NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeSt return out, nil } +func (c *nodeClient) NodeExpandVolume(ctx context.Context, in *NodeExpandVolumeRequest, opts ...grpc.CallOption) (*NodeExpandVolumeResponse, error) { + out := new(NodeExpandVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeExpandVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *nodeClient) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) { out := new(NodeGetCapabilitiesResponse) err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetCapabilities", in, out, opts...) @@ -4905,6 +5305,7 @@ type NodeServer interface { NodePublishVolume(context.Context, *NodePublishVolumeRequest) (*NodePublishVolumeResponse, error) NodeUnpublishVolume(context.Context, *NodeUnpublishVolumeRequest) (*NodeUnpublishVolumeResponse, error) NodeGetVolumeStats(context.Context, *NodeGetVolumeStatsRequest) (*NodeGetVolumeStatsResponse, error) + NodeExpandVolume(context.Context, *NodeExpandVolumeRequest) (*NodeExpandVolumeResponse, error) NodeGetCapabilities(context.Context, *NodeGetCapabilitiesRequest) (*NodeGetCapabilitiesResponse, error) NodeGetInfo(context.Context, *NodeGetInfoRequest) (*NodeGetInfoResponse, error) } @@ -5003,6 +5404,24 @@ func _Node_NodeGetVolumeStats_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Node_NodeExpandVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeExpandVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeExpandVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeExpandVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeExpandVolume(ctx, req.(*NodeExpandVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Node_NodeGetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(NodeGetCapabilitiesRequest) if err := dec(in); err != nil { @@ -5063,6 +5482,10 @@ var _Node_serviceDesc = grpc.ServiceDesc{ MethodName: "NodeGetVolumeStats", Handler: _Node_NodeGetVolumeStats_Handler, }, + { + MethodName: "NodeExpandVolume", + Handler: _Node_NodeExpandVolume_Handler, + }, { MethodName: "NodeGetCapabilities", Handler: _Node_NodeGetCapabilities_Handler, @@ -5077,201 +5500,214 @@ var _Node_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_csi_1092d4f3f3c8dc30) + proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_csi_2c5455657a82ae49) } -var fileDescriptor_csi_1092d4f3f3c8dc30 = []byte{ - // 3070 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x1a, 0x4d, 0x6f, 0xe3, 0xc6, - 0xd5, 0xd4, 0x87, 0x3f, 0x9e, 0x6d, 0x45, 0x3b, 0xfe, 0x58, 0x2d, 0x6d, 0xef, 0x7a, 0xb9, 0xd9, - 0xc4, 0xd9, 0x64, 0xe5, 0xc6, 0xc9, 0x06, 0xcd, 0xee, 0xa6, 0x8d, 0x24, 0x2b, 0xb6, 0xb2, 0x5a, - 0xd9, 0xa1, 0x64, 0xa7, 0xbb, 0x6d, 0xc0, 0xd0, 0xd2, 0x58, 0x4b, 0x44, 0x22, 0x15, 0x72, 0xe4, - 0xae, 0x7b, 0x2b, 0x0a, 0xf4, 0xd2, 0x53, 0x4f, 0xed, 0xad, 0x40, 0x7a, 0x6a, 0xd1, 0xa0, 0xa7, - 0xa2, 0xc7, 0x02, 0xbd, 0x14, 0xe8, 0x1f, 0x68, 0x6f, 0xb9, 0x07, 0x2d, 0x10, 0xf4, 0xd0, 0x43, - 0x81, 0x02, 0x05, 0x39, 0x43, 0x8a, 0x43, 0x91, 0x94, 0xb4, 0xde, 0x20, 0x87, 0x9e, 0x24, 0xbe, - 0x79, 0x5f, 0xf3, 0xe6, 0xbd, 0x37, 0xef, 0x3d, 0x12, 0xee, 0xb4, 0x35, 0xf2, 0xa4, 0x7f, 0x92, - 0x6f, 0x1a, 0xdd, 0xed, 0xa6, 0xa1, 0x13, 0x55, 0xd3, 0xb1, 0x79, 0xdb, 0x22, 0x86, 0xa9, 0xb6, - 0xf1, 0x6d, 0x4d, 0x27, 0xd8, 0x3c, 0x55, 0x9b, 0x78, 0xdb, 0xea, 0xe1, 0xe6, 0x76, 0xd3, 0xd2, - 0xf2, 0x3d, 0xd3, 0x20, 0x06, 0x9a, 0xb6, 0xff, 0x9e, 0xbd, 0x2e, 0x6e, 0xb6, 0x0d, 0xa3, 0xdd, - 0xc1, 0xdb, 0x0e, 0xf4, 0xa4, 0x7f, 0xba, 0xdd, 0xc2, 0x56, 0xd3, 0xd4, 0x7a, 0xc4, 0x30, 0x29, - 0xa6, 0x78, 0x2d, 0x88, 0x41, 0xb4, 0x2e, 0xb6, 0x88, 0xda, 0xed, 0x31, 0x84, 0xab, 0x41, 0x84, - 0x1f, 0x9a, 0x6a, 0xaf, 0x87, 0x4d, 0x8b, 0xae, 0x4b, 0xab, 0xb0, 0xbc, 0x87, 0xc9, 0x61, 0xa7, - 0xdf, 0xd6, 0xf4, 0x8a, 0x7e, 0x6a, 0xc8, 0xf8, 0xd3, 0x3e, 0xb6, 0x88, 0xf4, 0x77, 0x01, 0x56, - 0x02, 0x0b, 0x56, 0xcf, 0xd0, 0x2d, 0x8c, 0x10, 0xa4, 0x74, 0xb5, 0x8b, 0x73, 0xc2, 0xa6, 0xb0, - 0x35, 0x27, 0x3b, 0xff, 0xd1, 0x4d, 0xc8, 0x9c, 0x61, 0xbd, 0x65, 0x98, 0xca, 0x19, 0x36, 0x2d, - 0xcd, 0xd0, 0x73, 0x09, 0x67, 0x75, 0x91, 0x42, 0x8f, 0x29, 0x10, 0xed, 0xc1, 0x6c, 0x57, 0xd5, - 0xb5, 0x53, 0x6c, 0x91, 0x5c, 0x72, 0x33, 0xb9, 0x35, 0xbf, 0xf3, 0x6a, 0x9e, 0x6e, 0x35, 0x1f, - 0x2a, 0x2b, 0xff, 0x90, 0x61, 0x97, 0x75, 0x62, 0x9e, 0xcb, 0x1e, 0xb1, 0x78, 0x0f, 0x16, 0xb9, - 0x25, 0x94, 0x85, 0xe4, 0x27, 0xf8, 0x9c, 0xe9, 0x64, 0xff, 0x45, 0xcb, 0x90, 0x3e, 0x53, 0x3b, - 0x7d, 0xcc, 0x34, 0xa1, 0x0f, 0x77, 0x13, 0xdf, 0x16, 0xa4, 0xab, 0xb0, 0xee, 0x49, 0x2b, 0xa9, - 0x3d, 0xf5, 0x44, 0xeb, 0x68, 0x44, 0xc3, 0x96, 0xbb, 0xf5, 0x8f, 0x60, 0x23, 0x62, 0x9d, 0x59, - 0xe0, 0x3e, 0x2c, 0x34, 0x7d, 0xf0, 0x9c, 0xe0, 0x6c, 0x25, 0xe7, 0x6e, 0x25, 0x40, 0x79, 0x2e, - 0x73, 0xd8, 0xd2, 0xbf, 0x04, 0xc8, 0x06, 0x51, 0xd0, 0x7d, 0x98, 0xb1, 0xb0, 0x79, 0xa6, 0x35, - 0xa9, 0x5d, 0xe7, 0x77, 0x36, 0xa3, 0xb8, 0xe5, 0xeb, 0x14, 0x6f, 0x7f, 0x4a, 0x76, 0x49, 0xc4, - 0x5f, 0x08, 0x30, 0xc3, 0xc0, 0xe8, 0x6d, 0x48, 0x91, 0xf3, 0x1e, 0x65, 0x93, 0xd9, 0xb9, 0x39, - 0x8a, 0x4d, 0xbe, 0x71, 0xde, 0xc3, 0xb2, 0x43, 0x22, 0x7d, 0x00, 0x29, 0xfb, 0x09, 0xcd, 0xc3, - 0xcc, 0x51, 0xed, 0x41, 0xed, 0xe0, 0xc3, 0x5a, 0x76, 0x0a, 0xad, 0x02, 0x2a, 0x1d, 0xd4, 0x1a, - 0xf2, 0x41, 0xb5, 0x5a, 0x96, 0x95, 0x7a, 0x59, 0x3e, 0xae, 0x94, 0xca, 0x59, 0x01, 0xbd, 0x08, - 0x9b, 0xc7, 0x07, 0xd5, 0xa3, 0x87, 0x65, 0xa5, 0x50, 0x2a, 0x95, 0xeb, 0xf5, 0x4a, 0xb1, 0x52, - 0xad, 0x34, 0x1e, 0x29, 0xa5, 0x83, 0x5a, 0xbd, 0x21, 0x17, 0x2a, 0xb5, 0x46, 0x3d, 0x9b, 0x28, - 0x4e, 0x53, 0x6d, 0xa4, 0x0c, 0x2c, 0x1c, 0x9a, 0xc6, 0x09, 0x76, 0x6d, 0x5c, 0x80, 0x45, 0xf6, - 0xcc, 0x6c, 0xfa, 0x2d, 0x48, 0x9b, 0x58, 0x6d, 0x9d, 0xb3, 0xed, 0x8b, 0x79, 0xea, 0xb7, 0x79, - 0xd7, 0x6f, 0xf3, 0x45, 0xc3, 0xe8, 0x1c, 0xdb, 0x67, 0x28, 0x53, 0x44, 0xe9, 0xab, 0x14, 0x2c, - 0x95, 0x4c, 0xac, 0x12, 0x7c, 0x6c, 0x74, 0xfa, 0x5d, 0x97, 0x75, 0xa8, 0x7f, 0xde, 0x87, 0x8c, - 0x7d, 0x06, 0x4d, 0x8d, 0x9c, 0x2b, 0xa6, 0xaa, 0xb7, 0xa9, 0x57, 0xcc, 0xef, 0xac, 0xb8, 0xe6, - 0x29, 0xb1, 0x55, 0xd9, 0x5e, 0x94, 0x17, 0x9b, 0xfe, 0x47, 0x54, 0x81, 0xa5, 0x33, 0x47, 0x84, - 0xc2, 0x1d, 0x7b, 0x92, 0x3f, 0x76, 0xaa, 0x85, 0xef, 0xd8, 0xd1, 0x19, 0x0f, 0xd1, 0xb0, 0x85, - 0x1e, 0x00, 0xf4, 0x54, 0x53, 0xed, 0x62, 0x82, 0x4d, 0x2b, 0x97, 0xe2, 0x63, 0x20, 0x64, 0x37, - 0xf9, 0x43, 0x0f, 0x9b, 0xc6, 0x80, 0x8f, 0x1c, 0xed, 0xd9, 0x4e, 0xd3, 0x34, 0x31, 0xb1, 0x72, - 0x69, 0x87, 0xd3, 0x56, 0x1c, 0xa7, 0x3a, 0x45, 0x75, 0xd8, 0x14, 0x93, 0xbf, 0x2c, 0x0a, 0xb2, - 0x4b, 0x8d, 0x0e, 0x60, 0xc5, 0xdd, 0xa0, 0xa1, 0x13, 0xac, 0x13, 0xc5, 0x32, 0xfa, 0x66, 0x13, - 0xe7, 0xa6, 0x1d, 0x2b, 0xad, 0x05, 0xb6, 0x48, 0x71, 0xea, 0x0e, 0x8a, 0xcc, 0x4c, 0xc3, 0x01, - 0xd1, 0x63, 0x10, 0xd5, 0x66, 0x13, 0x5b, 0x96, 0x46, 0x6d, 0xa1, 0x98, 0xf8, 0xd3, 0xbe, 0x66, - 0xe2, 0x2e, 0xd6, 0x89, 0x95, 0x9b, 0xe1, 0xb9, 0x36, 0x8c, 0x9e, 0xd1, 0x31, 0xda, 0xe7, 0xf2, - 0x00, 0x47, 0xbe, 0xc2, 0x91, 0xfb, 0x56, 0x2c, 0xf1, 0x1d, 0x78, 0x21, 0x60, 0x94, 0x49, 0xa2, - 0x5f, 0xbc, 0x0b, 0x0b, 0x7e, 0x4b, 0x4c, 0x94, 0x39, 0x7e, 0x96, 0x80, 0xa5, 0x10, 0x1b, 0xa0, - 0x7d, 0x98, 0xb5, 0x74, 0xb5, 0x67, 0x3d, 0x31, 0x08, 0xf3, 0xdf, 0x5b, 0x31, 0x26, 0xcb, 0xd7, - 0x19, 0x2e, 0x7d, 0xdc, 0x9f, 0x92, 0x3d, 0x6a, 0x54, 0x84, 0x69, 0x6a, 0x4f, 0xe6, 0xa0, 0x5b, - 0x71, 0x7c, 0x28, 0xcc, 0xe3, 0xc2, 0x28, 0xc5, 0xd7, 0x21, 0xc3, 0x4b, 0x40, 0xd7, 0x60, 0xde, - 0x95, 0xa0, 0x68, 0x2d, 0xb6, 0x57, 0x70, 0x41, 0x95, 0x96, 0xf8, 0x2a, 0x2c, 0xf8, 0x99, 0xa1, - 0x35, 0x98, 0x63, 0x0e, 0xe1, 0xa1, 0xcf, 0x52, 0x40, 0xa5, 0xe5, 0xc5, 0xf4, 0x77, 0x60, 0x99, - 0xf7, 0x33, 0x16, 0xca, 0x2f, 0x79, 0x7b, 0xa0, 0xb6, 0xc8, 0xf0, 0x7b, 0x70, 0xf5, 0x94, 0x7e, - 0x9b, 0x82, 0x6c, 0x30, 0x68, 0xd0, 0x7d, 0x48, 0x9f, 0x74, 0x8c, 0xe6, 0x27, 0x8c, 0xf6, 0xc5, - 0xa8, 0xe8, 0xca, 0x17, 0x6d, 0x2c, 0x0a, 0xdd, 0x9f, 0x92, 0x29, 0x91, 0x4d, 0xdd, 0x35, 0xfa, - 0x3a, 0x61, 0xd6, 0x8b, 0xa6, 0x7e, 0x68, 0x63, 0x0d, 0xa8, 0x1d, 0x22, 0xb4, 0x0b, 0xf3, 0xd4, - 0xed, 0x94, 0xae, 0xd1, 0xc2, 0xb9, 0xa4, 0xc3, 0xe3, 0x46, 0x24, 0x8f, 0x82, 0x83, 0xfb, 0xd0, - 0x68, 0x61, 0x19, 0x54, 0xef, 0xbf, 0xb8, 0x08, 0xf3, 0x3e, 0xdd, 0xc4, 0x3d, 0x98, 0xf7, 0x09, - 0x43, 0x97, 0x61, 0xe6, 0xd4, 0x52, 0xbc, 0x0c, 0x3d, 0x27, 0x4f, 0x9f, 0x5a, 0x4e, 0xd2, 0xbd, - 0x06, 0xf3, 0x8e, 0x16, 0xca, 0x69, 0x47, 0x6d, 0x5b, 0xb9, 0xc4, 0x66, 0xd2, 0x3e, 0x23, 0x07, - 0xf4, 0x9e, 0x0d, 0x11, 0xff, 0x21, 0x00, 0x0c, 0x44, 0xa2, 0xfb, 0x90, 0x72, 0xb4, 0xa4, 0x79, - 0x7e, 0x6b, 0x0c, 0x2d, 0xf3, 0x8e, 0xaa, 0x0e, 0x95, 0xf4, 0x2b, 0x01, 0x52, 0x0e, 0x9b, 0x60, - 0xae, 0xaf, 0x57, 0x6a, 0x7b, 0xd5, 0xb2, 0x52, 0x3b, 0xd8, 0x2d, 0x2b, 0x1f, 0xca, 0x95, 0x46, - 0x59, 0xce, 0x0a, 0x68, 0x0d, 0x2e, 0xfb, 0xe1, 0x72, 0xb9, 0xb0, 0x5b, 0x96, 0x95, 0x83, 0x5a, - 0xf5, 0x51, 0x36, 0x81, 0x44, 0x58, 0x7d, 0x78, 0x54, 0x6d, 0x54, 0x86, 0xd7, 0x92, 0x68, 0x1d, - 0x72, 0xbe, 0x35, 0xc6, 0x83, 0xb1, 0x4d, 0xd9, 0x6c, 0x7d, 0xab, 0xf4, 0x2f, 0x5b, 0x4c, 0x17, - 0x17, 0xbd, 0xc3, 0x70, 0x9c, 0xed, 0x43, 0x58, 0xe4, 0x72, 0xb4, 0x5d, 0x72, 0xb0, 0xa4, 0xd2, - 0x52, 0x4e, 0xce, 0x89, 0x73, 0x0d, 0x0b, 0x5b, 0x49, 0x79, 0xd1, 0x85, 0x16, 0x6d, 0xa0, 0x6d, - 0xd6, 0x8e, 0xd6, 0xd5, 0x08, 0xc3, 0x49, 0x38, 0x38, 0xe0, 0x80, 0x1c, 0x04, 0xe9, 0x8b, 0x04, - 0x4c, 0xb3, 0xb3, 0xb9, 0xe9, 0xbb, 0x25, 0x38, 0x96, 0x2e, 0x94, 0xb2, 0xe4, 0x82, 0x23, 0xc1, - 0x07, 0x07, 0xda, 0x87, 0x8c, 0x3f, 0x95, 0x3e, 0x75, 0x0b, 0x9d, 0xeb, 0xfc, 0x01, 0xf9, 0xe3, - 0xf9, 0x29, 0x2b, 0x6f, 0x16, 0xcf, 0xfc, 0x30, 0x54, 0x84, 0x4c, 0x20, 0x1b, 0xa7, 0x46, 0x67, - 0xe3, 0xc5, 0x26, 0x97, 0x98, 0x0a, 0xb0, 0xe4, 0x26, 0xd2, 0x0e, 0x56, 0x08, 0x4b, 0xb4, 0xec, - 0xb6, 0xc8, 0x0e, 0x25, 0x60, 0x34, 0x40, 0x76, 0x61, 0xe2, 0xbb, 0x80, 0x86, 0x75, 0x9d, 0x28, - 0x6b, 0xf6, 0x61, 0x29, 0x24, 0xc5, 0xa3, 0x3c, 0xcc, 0x39, 0x47, 0x65, 0x69, 0x04, 0xb3, 0x12, - 0x6a, 0x58, 0xa3, 0x01, 0x8a, 0x8d, 0xdf, 0x33, 0xf1, 0x29, 0x36, 0x4d, 0xdc, 0x72, 0xc2, 0x23, - 0x14, 0xdf, 0x43, 0x91, 0x7e, 0x22, 0xc0, 0xac, 0x0b, 0x47, 0x77, 0x61, 0xd6, 0xc2, 0x6d, 0x7a, - 0xfd, 0x50, 0x59, 0x57, 0x83, 0xb4, 0xf9, 0x3a, 0x43, 0x60, 0xc5, 0xa6, 0x8b, 0x6f, 0x17, 0x9b, - 0xdc, 0xd2, 0x44, 0x9b, 0xff, 0xa3, 0x00, 0x4b, 0xbb, 0xb8, 0x83, 0x83, 0x55, 0x4a, 0x5c, 0x86, - 0xf5, 0x5f, 0xec, 0x09, 0xfe, 0x62, 0x0f, 0x61, 0x15, 0x73, 0xb1, 0x5f, 0xe8, 0xb2, 0x5b, 0x85, - 0x65, 0x5e, 0x1a, 0x4d, 0xef, 0xd2, 0x3f, 0x93, 0x70, 0xd5, 0xf6, 0x05, 0xd3, 0xe8, 0x74, 0xb0, - 0x79, 0xd8, 0x3f, 0xe9, 0x68, 0xd6, 0x93, 0x09, 0x36, 0x77, 0x19, 0x66, 0x74, 0xa3, 0xe5, 0x0b, - 0x9e, 0x69, 0xfb, 0xb1, 0xd2, 0x42, 0x65, 0xb8, 0x14, 0x2c, 0xb3, 0xce, 0x59, 0x12, 0x8e, 0x2e, - 0xb2, 0xb2, 0x67, 0xc1, 0x1b, 0x44, 0x84, 0x59, 0xbb, 0x40, 0x34, 0xf4, 0xce, 0xb9, 0x13, 0x31, - 0xb3, 0xb2, 0xf7, 0x8c, 0xe4, 0x60, 0xc5, 0xf4, 0x86, 0x57, 0x31, 0xc5, 0xee, 0x28, 0xae, 0x78, - 0xfa, 0x78, 0x28, 0xe2, 0xa7, 0x1d, 0xd6, 0x6f, 0x8f, 0xc9, 0x7a, 0x64, 0x26, 0xb8, 0xc8, 0x29, - 0x3e, 0x87, 0xf0, 0xfd, 0xab, 0x00, 0xd7, 0x22, 0xb7, 0xc0, 0xae, 0xfc, 0x16, 0xbc, 0xd0, 0xa3, - 0x0b, 0x9e, 0x11, 0x68, 0x94, 0xdd, 0x1b, 0x69, 0x04, 0xd6, 0xe9, 0x31, 0x28, 0x67, 0x86, 0x4c, - 0x8f, 0x03, 0x8a, 0x05, 0x58, 0x0a, 0x41, 0x9b, 0x68, 0x33, 0x5f, 0x0a, 0xb0, 0x39, 0x50, 0xe5, - 0x48, 0xef, 0x3d, 0x3f, 0xf7, 0x6d, 0x0c, 0x7c, 0x8b, 0xa6, 0xfc, 0x3b, 0xc3, 0x7b, 0x0f, 0x17, - 0xf8, 0x75, 0x45, 0xf0, 0x0d, 0xb8, 0x1e, 0x23, 0x9a, 0x85, 0xf3, 0x17, 0x29, 0xb8, 0x7e, 0xac, - 0x76, 0xb4, 0x96, 0x57, 0xc8, 0x85, 0xf4, 0xc4, 0xf1, 0x26, 0x69, 0x0e, 0x45, 0x00, 0xcd, 0x5a, - 0xf7, 0xbd, 0xa8, 0x1d, 0xc5, 0x7f, 0x8c, 0xeb, 0xf0, 0x39, 0x36, 0x61, 0x8f, 0x42, 0x9a, 0xb0, - 0xb7, 0xc7, 0xd7, 0x35, 0xae, 0x25, 0x3b, 0x0a, 0x26, 0x98, 0xb7, 0xc6, 0xe7, 0x1b, 0xe3, 0x05, - 0x17, 0x8e, 0xe2, 0x6f, 0xb2, 0x6b, 0xfa, 0x73, 0x0a, 0xa4, 0xb8, 0xdd, 0xb3, 0x1c, 0x22, 0xc3, - 0x5c, 0xd3, 0xd0, 0x4f, 0x35, 0xb3, 0x8b, 0x5b, 0xac, 0xfa, 0x7f, 0x73, 0x1c, 0xe3, 0xb1, 0x04, - 0x52, 0x72, 0x69, 0xe5, 0x01, 0x1b, 0x94, 0x83, 0x99, 0x2e, 0xb6, 0x2c, 0xb5, 0xed, 0xaa, 0xe5, - 0x3e, 0x8a, 0x9f, 0x27, 0x61, 0xce, 0x23, 0x41, 0xfa, 0x90, 0x07, 0xd3, 0xf4, 0xb5, 0xf7, 0x2c, - 0x0a, 0x3c, 0xbb, 0x33, 0x27, 0x9e, 0xc1, 0x99, 0x5b, 0x9c, 0x33, 0xd3, 0x70, 0xd8, 0x7d, 0x26, - 0xb5, 0x63, 0xfc, 0xfa, 0x1b, 0x77, 0x40, 0xe9, 0x07, 0x80, 0xaa, 0x9a, 0xc5, 0xba, 0x28, 0x2f, - 0x2d, 0xd9, 0x4d, 0x93, 0xfa, 0x54, 0xc1, 0x3a, 0x31, 0x35, 0x56, 0xae, 0xa7, 0x65, 0xe8, 0xaa, - 0x4f, 0xcb, 0x14, 0x62, 0x97, 0xf4, 0x16, 0x51, 0x4d, 0xa2, 0xe9, 0x6d, 0x85, 0x18, 0x9f, 0x60, - 0x6f, 0x30, 0xe9, 0x42, 0x1b, 0x36, 0x50, 0xfa, 0x4c, 0x80, 0x25, 0x8e, 0x3d, 0xf3, 0xc9, 0x7b, - 0x30, 0x33, 0xe0, 0xcd, 0x95, 0xf1, 0x21, 0xd8, 0x79, 0x6a, 0x36, 0x97, 0x02, 0x6d, 0x00, 0xe8, - 0xf8, 0x29, 0xe1, 0xe4, 0xce, 0xd9, 0x10, 0x47, 0xa6, 0xb8, 0x0d, 0x69, 0x6a, 0x86, 0x71, 0xfb, - 0xe5, 0xcf, 0x13, 0x80, 0xf6, 0x30, 0xf1, 0xda, 0x20, 0x66, 0x83, 0x08, 0x5f, 0x12, 0x9e, 0xc1, - 0x97, 0xde, 0xe7, 0x7c, 0x89, 0x7a, 0xe3, 0x2d, 0xdf, 0x84, 0x36, 0x20, 0x3a, 0x36, 0x13, 0x46, - 0xb4, 0x1e, 0xb4, 0x9e, 0x1b, 0xaf, 0xf5, 0xb8, 0xa0, 0xcb, 0xec, 0xc2, 0x12, 0xa7, 0x33, 0x3b, - 0xd3, 0xdb, 0x80, 0xd4, 0x33, 0x55, 0xeb, 0xa8, 0xb6, 0x5e, 0x6e, 0x67, 0xc7, 0x3a, 0xbd, 0x4b, - 0xde, 0x8a, 0x4b, 0x26, 0x49, 0xfe, 0x82, 0x81, 0xf1, 0x0b, 0x4e, 0x8c, 0x3b, 0xfe, 0x8b, 0x76, - 0x08, 0x87, 0xc9, 0xdd, 0x0b, 0x9d, 0x1a, 0xdf, 0x18, 0x2e, 0x12, 0xd8, 0x64, 0x36, 0x72, 0x80, - 0xfc, 0xef, 0x04, 0xac, 0xc5, 0x60, 0xa3, 0x7b, 0x90, 0x34, 0x7b, 0x4d, 0xe6, 0x4c, 0x2f, 0x8f, - 0xc1, 0x3f, 0x2f, 0x1f, 0x96, 0xf6, 0xa7, 0x64, 0x9b, 0x4a, 0xfc, 0x79, 0x02, 0x92, 0xf2, 0x61, - 0x09, 0xbd, 0xcb, 0x8d, 0x91, 0x5f, 0x1b, 0x93, 0x8b, 0x7f, 0x9a, 0xfc, 0x17, 0x21, 0x6c, 0x9c, - 0x9c, 0x83, 0xe5, 0x92, 0x5c, 0x2e, 0x34, 0xca, 0xca, 0x6e, 0xb9, 0x5a, 0x6e, 0x94, 0x15, 0x3a, - 0x44, 0xce, 0x0a, 0x68, 0x1d, 0x72, 0x87, 0x47, 0xc5, 0x6a, 0xa5, 0xbe, 0xaf, 0x1c, 0xd5, 0xdc, - 0x7f, 0x6c, 0x35, 0x81, 0xb2, 0xb0, 0x50, 0xad, 0xd4, 0x1b, 0x0c, 0x50, 0xcf, 0x26, 0x6d, 0xc8, - 0x5e, 0xb9, 0xa1, 0x94, 0x0a, 0x87, 0x85, 0x52, 0xa5, 0xf1, 0x28, 0x9b, 0x42, 0x22, 0xac, 0xf2, - 0xbc, 0xeb, 0xb5, 0xc2, 0x61, 0x7d, 0xff, 0xa0, 0x91, 0x4d, 0x23, 0x04, 0x19, 0x87, 0xde, 0x05, - 0xd5, 0xb3, 0xd3, 0x36, 0x87, 0x52, 0xf5, 0xa0, 0xe6, 0xe9, 0x30, 0x83, 0x96, 0x21, 0xeb, 0x4a, - 0x96, 0xcb, 0x85, 0x5d, 0x67, 0x8a, 0x31, 0xeb, 0x0d, 0xbc, 0xbe, 0x4c, 0xc0, 0x0a, 0x9d, 0x78, - 0xb9, 0xf3, 0x35, 0x37, 0x06, 0xb7, 0x20, 0x4b, 0x7b, 0x74, 0x25, 0x58, 0x25, 0x65, 0x28, 0xfc, - 0xd8, 0xad, 0x95, 0xdc, 0xe9, 0x74, 0xc2, 0x37, 0x9d, 0xae, 0x04, 0x2b, 0xc7, 0x5b, 0xfc, 0x1c, - 0x37, 0x20, 0x2d, 0xae, 0x19, 0x79, 0x18, 0x52, 0xda, 0xdc, 0x8e, 0xe7, 0x16, 0x97, 0xf6, 0x2f, - 0xd2, 0x79, 0x5c, 0x30, 0x7a, 0xdf, 0x83, 0xd5, 0xa0, 0xbe, 0x2c, 0x90, 0x5e, 0x1b, 0x9a, 0xb6, - 0x7a, 0xe9, 0xc4, 0xc3, 0xf5, 0x30, 0xa4, 0xbf, 0x09, 0x30, 0xeb, 0x82, 0xed, 0x94, 0x6c, 0x69, - 0x3f, 0xc2, 0xdc, 0x74, 0x67, 0xce, 0x86, 0x78, 0xc3, 0x22, 0xff, 0x9c, 0x34, 0x11, 0x9c, 0x93, - 0x86, 0x9e, 0x73, 0x32, 0xf4, 0x9c, 0xbf, 0x0b, 0x8b, 0x4d, 0x5b, 0x7d, 0xcd, 0xd0, 0x15, 0xa2, - 0x75, 0xdd, 0xe1, 0xcd, 0xf0, 0x7b, 0x8d, 0x86, 0xfb, 0xc2, 0x4e, 0x5e, 0x70, 0x09, 0x6c, 0x10, - 0xda, 0x84, 0x05, 0xe7, 0x3d, 0x87, 0x42, 0x0c, 0xa5, 0x6f, 0xe1, 0x5c, 0xda, 0x69, 0x65, 0xc1, - 0x81, 0x35, 0x8c, 0x23, 0x0b, 0x4b, 0x7f, 0x12, 0x60, 0x85, 0x76, 0xe8, 0x41, 0x77, 0x1c, 0x35, - 0xef, 0xf5, 0x7b, 0x5c, 0x20, 0xcb, 0x87, 0x32, 0xfc, 0xba, 0x1a, 0x94, 0x1c, 0xac, 0x06, 0xe5, - 0xb1, 0xae, 0xe4, 0x37, 0x02, 0x2c, 0xdb, 0x57, 0xac, 0xbb, 0xf0, 0xbc, 0x6f, 0xfc, 0x09, 0x4e, - 0x32, 0x60, 0xcc, 0x54, 0xd0, 0x98, 0xd2, 0xef, 0x04, 0x58, 0x09, 0xe8, 0xca, 0x3c, 0xf5, 0x9d, - 0x60, 0xf9, 0x70, 0xc3, 0x5f, 0x3e, 0x0c, 0xe1, 0x4f, 0x58, 0x40, 0xdc, 0x71, 0x0b, 0x88, 0xc9, - 0x02, 0xe2, 0xab, 0x14, 0xac, 0xd6, 0x8c, 0x16, 0xae, 0x13, 0xb5, 0x3d, 0xc9, 0x50, 0xea, 0xfb, - 0xc3, 0x3d, 0x3e, 0xf5, 0x9d, 0x1d, 0x57, 0x58, 0x38, 0xd7, 0x71, 0x5a, 0x7b, 0x94, 0x87, 0x25, - 0x8b, 0xa8, 0x6d, 0xe7, 0xd0, 0x54, 0xb3, 0x8d, 0x89, 0xd2, 0x53, 0xc9, 0x13, 0x76, 0x22, 0x97, - 0xd8, 0x52, 0xc3, 0x59, 0x39, 0x54, 0xc9, 0x93, 0xf0, 0x59, 0x51, 0x6a, 0xe2, 0x59, 0xd1, 0xfb, - 0xc1, 0x76, 0xed, 0xd5, 0x11, 0x7b, 0x89, 0x49, 0xbd, 0xdf, 0x8b, 0x98, 0x03, 0xbd, 0x3e, 0x82, - 0xe5, 0xe8, 0xf9, 0xcf, 0xc5, 0xe7, 0x1e, 0xdf, 0xf0, 0x08, 0xe9, 0x0a, 0x5c, 0x1e, 0xda, 0x3c, - 0x0b, 0xf4, 0x36, 0xe4, 0xec, 0xa5, 0x23, 0xdd, 0x9a, 0xd0, 0x1d, 0x23, 0x3c, 0x26, 0x11, 0xe1, - 0x31, 0xd2, 0x1a, 0x5c, 0x09, 0x11, 0xc4, 0xb4, 0xf8, 0x43, 0x9a, 0xaa, 0x31, 0xf9, 0x34, 0xf3, - 0xa3, 0xa8, 0xa8, 0x78, 0xd3, 0x7f, 0xec, 0xa1, 0x83, 0xbf, 0xaf, 0x23, 0x2e, 0xae, 0xc1, 0xbc, - 0x1f, 0x8f, 0x25, 0x2b, 0x32, 0x22, 0x70, 0xd2, 0x17, 0x1a, 0xb2, 0x4e, 0x07, 0x86, 0xac, 0xd5, - 0x41, 0x50, 0xcd, 0xf0, 0x05, 0x48, 0xa4, 0x29, 0x62, 0xc2, 0xea, 0xf1, 0x50, 0x58, 0xcd, 0xf2, - 0x93, 0xdb, 0x48, 0xa6, 0xff, 0x07, 0x81, 0xc5, 0x9c, 0x3a, 0x74, 0xa4, 0x2a, 0x3d, 0x06, 0x91, - 0x7a, 0xfc, 0xe4, 0x43, 0xce, 0x80, 0x1b, 0x25, 0x82, 0x6e, 0x24, 0x6d, 0xc0, 0x5a, 0x28, 0x6f, - 0x26, 0xfa, 0x11, 0xd5, 0x6b, 0x0f, 0xb3, 0x1e, 0xb9, 0x4e, 0x54, 0x62, 0x8d, 0x2b, 0x99, 0x2d, - 0xfa, 0x25, 0x53, 0x90, 0x23, 0x79, 0x8f, 0xee, 0x2a, 0xc8, 0x9a, 0xdd, 0xb8, 0xaf, 0x40, 0xba, - 0xef, 0x8c, 0x7b, 0xe8, 0x7d, 0xbb, 0xc4, 0xbb, 0xf4, 0x91, 0xbd, 0x24, 0x53, 0x0c, 0xe9, 0xf7, - 0x02, 0xcc, 0xfb, 0xc0, 0x68, 0x1d, 0xe6, 0xbc, 0xee, 0xcf, 0x2d, 0x0d, 0x3d, 0x80, 0x7d, 0x06, - 0xc4, 0x20, 0x6a, 0x87, 0xbd, 0x41, 0xa4, 0x0f, 0x76, 0x35, 0xdf, 0xb7, 0x30, 0xad, 0x1c, 0x92, - 0xb2, 0xf3, 0x1f, 0xbd, 0x06, 0xa9, 0xbe, 0xae, 0x11, 0x27, 0xf6, 0x32, 0xc1, 0xa0, 0x72, 0x44, - 0xe5, 0x8f, 0x74, 0x8d, 0xc8, 0x0e, 0x96, 0x74, 0x0b, 0x52, 0xf6, 0x13, 0xdf, 0x24, 0xcd, 0x41, - 0xba, 0xf8, 0xa8, 0x51, 0xae, 0x67, 0x05, 0x04, 0x30, 0x5d, 0xa9, 0x1d, 0xec, 0x96, 0xeb, 0xd9, - 0x84, 0xb4, 0xee, 0x6d, 0x3d, 0xac, 0x09, 0xfd, 0x98, 0x1e, 0x49, 0x54, 0xfb, 0x59, 0x08, 0x6d, - 0x3f, 0x37, 0xb8, 0xcb, 0x69, 0x44, 0xe3, 0xf9, 0x85, 0x00, 0x2b, 0xa1, 0x78, 0xe8, 0x8e, 0xbf, - 0xe5, 0xbc, 0x1e, 0xcb, 0xd3, 0xdf, 0x6c, 0xfe, 0x54, 0xa0, 0xcd, 0xe6, 0x5d, 0xae, 0xd9, 0x7c, - 0x69, 0x24, 0xbd, 0xbf, 0xcd, 0x2c, 0x45, 0x74, 0x99, 0xf5, 0x46, 0x61, 0xaf, 0xac, 0x1c, 0xd5, - 0xe8, 0xaf, 0xd7, 0x65, 0x2e, 0x43, 0xd6, 0xee, 0x1a, 0xd9, 0xa7, 0x4b, 0xf5, 0x46, 0x81, 0xfb, - 0x4c, 0x69, 0x19, 0x10, 0xb3, 0xa1, 0xff, 0x5b, 0xb8, 0xcf, 0x04, 0x58, 0xe2, 0xc0, 0xcc, 0xa4, - 0xbe, 0x57, 0x01, 0x02, 0xf7, 0x2a, 0x60, 0x1b, 0x96, 0xed, 0x22, 0x95, 0x7a, 0xad, 0xa5, 0xf4, - 0xb0, 0xa9, 0xd8, 0x2b, 0xcc, 0x77, 0x2e, 0x75, 0xd5, 0xa7, 0x6c, 0x74, 0x74, 0x88, 0x4d, 0x9b, - 0xf1, 0x73, 0x18, 0x96, 0xec, 0xfc, 0x47, 0x80, 0xd9, 0x4a, 0x0b, 0xeb, 0xc4, 0x3e, 0x8f, 0x1a, - 0x2c, 0x72, 0x1f, 0xd4, 0xa1, 0xf5, 0x88, 0xef, 0xec, 0x9c, 0x0d, 0x8a, 0x1b, 0xb1, 0x5f, 0xe1, - 0x49, 0x53, 0xe8, 0xd4, 0xf7, 0x31, 0x20, 0x37, 0x31, 0x7a, 0x71, 0x88, 0x32, 0xc4, 0x35, 0xc5, - 0x9b, 0x23, 0xb0, 0x3c, 0x39, 0x6f, 0x41, 0xda, 0xf9, 0x2c, 0x0c, 0x2d, 0x7b, 0xdf, 0xad, 0xf9, - 0xbe, 0x1a, 0x13, 0x57, 0x02, 0x50, 0x97, 0x6e, 0xe7, 0xbf, 0x33, 0x00, 0x83, 0xd1, 0x04, 0x7a, - 0x00, 0x0b, 0xfe, 0x2f, 0x53, 0xd0, 0x5a, 0xcc, 0x77, 0x51, 0xe2, 0x7a, 0xf8, 0xa2, 0xa7, 0xd3, - 0x03, 0x58, 0xf0, 0xbf, 0x07, 0x1d, 0x30, 0x0b, 0x79, 0x17, 0x3b, 0x60, 0x16, 0xfa, 0xea, 0x74, - 0x0a, 0x75, 0xe0, 0x72, 0xc4, 0x9b, 0x30, 0xf4, 0xd2, 0x78, 0xef, 0x0b, 0xc5, 0x97, 0xc7, 0x7c, - 0xa5, 0x26, 0x4d, 0x21, 0x13, 0xae, 0x44, 0xbe, 0x00, 0x42, 0x5b, 0xe3, 0xbe, 0x9e, 0x12, 0x5f, - 0x19, 0x03, 0xd3, 0x93, 0xd9, 0x07, 0x31, 0x7a, 0xea, 0x8c, 0x5e, 0x19, 0xfb, 0x75, 0x88, 0x78, - 0x6b, 0xfc, 0x21, 0xb6, 0x34, 0x85, 0xf6, 0x61, 0xde, 0x37, 0x92, 0x45, 0x62, 0xe8, 0x9c, 0x96, - 0x32, 0x5e, 0x8b, 0x99, 0xe1, 0x52, 0x4e, 0xbe, 0xb1, 0xe1, 0x80, 0xd3, 0xf0, 0xfc, 0x73, 0xc0, - 0x29, 0x64, 0xce, 0x18, 0x34, 0x7f, 0x20, 0x2f, 0x87, 0x99, 0x3f, 0x3c, 0xb1, 0x87, 0x99, 0x3f, - 0x22, 0xc9, 0x4b, 0x53, 0xe8, 0x03, 0xc8, 0xf0, 0x63, 0x13, 0xb4, 0x11, 0x3b, 0xfe, 0x11, 0xaf, - 0x46, 0x2d, 0xfb, 0x59, 0xf2, 0x5d, 0xfa, 0x80, 0x65, 0xe8, 0xb4, 0x60, 0xc0, 0x32, 0xa2, 0xb9, - 0x9f, 0xb2, 0xf3, 0x13, 0xd7, 0x01, 0x0f, 0xf2, 0x53, 0x58, 0xd3, 0x3f, 0xc8, 0x4f, 0xa1, 0x6d, - 0xb3, 0x34, 0xb5, 0xf3, 0xe3, 0x34, 0xa4, 0x9c, 0x44, 0xda, 0x80, 0x17, 0x02, 0x9d, 0x06, 0xba, - 0x1a, 0xdf, 0x7f, 0x89, 0xd7, 0x22, 0xd7, 0x3d, 0x75, 0x1f, 0xc3, 0xa5, 0xa1, 0xde, 0x01, 0x6d, - 0xfa, 0xe9, 0xc2, 0xfa, 0x17, 0xf1, 0x7a, 0x0c, 0x46, 0x90, 0x37, 0x9f, 0x0b, 0x36, 0x47, 0x15, - 0xb7, 0x3c, 0xef, 0xa8, 0xf8, 0xff, 0x98, 0xde, 0x5b, 0xc1, 0xc8, 0x97, 0x78, 0xbd, 0x42, 0x63, - 0xfe, 0x46, 0x2c, 0x8e, 0x27, 0xe1, 0x23, 0xef, 0xc2, 0xf4, 0x55, 0x63, 0x88, 0x53, 0x2e, 0xb4, - 0x08, 0x14, 0xa5, 0x38, 0x94, 0xe0, 0x06, 0x82, 0xb1, 0x13, 0x24, 0x0e, 0x8b, 0x9a, 0x1b, 0xb1, - 0x38, 0xfe, 0x68, 0xf7, 0x5d, 0xed, 0x83, 0x68, 0x1f, 0x2e, 0x03, 0x06, 0xd1, 0x1e, 0x52, 0x0b, - 0x48, 0x53, 0x77, 0xdf, 0x01, 0x68, 0x5a, 0x9a, 0x42, 0xfb, 0x16, 0xb4, 0x31, 0x34, 0xe8, 0x7b, - 0x4f, 0xc3, 0x9d, 0xd6, 0x41, 0x8f, 0x68, 0x86, 0x6e, 0xe5, 0x7e, 0x3d, 0xeb, 0x34, 0x4d, 0x73, - 0x4d, 0x4b, 0xa3, 0xed, 0x43, 0x31, 0xfd, 0x38, 0xd9, 0xb4, 0xb4, 0x93, 0x69, 0x07, 0xff, 0x8d, - 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x47, 0x5d, 0xbf, 0x76, 0x3a, 0x30, 0x00, 0x00, +var fileDescriptor_csi_2c5455657a82ae49 = []byte{ + // 3276 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5a, 0x4d, 0x70, 0xdb, 0xc6, + 0x15, 0x26, 0xf8, 0x23, 0x51, 0x4f, 0x3f, 0xa6, 0x57, 0x3f, 0xa6, 0x21, 0xc9, 0x96, 0xe1, 0xd8, + 0x51, 0x1c, 0x9b, 0x6a, 0x94, 0x38, 0xd3, 0xd8, 0x4e, 0x1b, 0x8a, 0xa2, 0x25, 0xc6, 0x34, 0xa9, + 0x80, 0x94, 0x1c, 0xbb, 0xcd, 0x20, 0x10, 0xb9, 0xa2, 0x31, 0x21, 0x01, 0x06, 0x00, 0x55, 0xa9, + 0x97, 0xce, 0xb4, 0xa7, 0x4c, 0xcf, 0x9d, 0xb6, 0xa7, 0xce, 0xa4, 0xbd, 0xb4, 0xd3, 0x4c, 0x4f, + 0x9d, 0x1e, 0x3b, 0xd3, 0x63, 0x0f, 0xbd, 0xb6, 0x93, 0x4b, 0xae, 0x9d, 0x4c, 0x3b, 0x93, 0xe9, + 0xb1, 0xa7, 0x0e, 0xb0, 0x0b, 0x10, 0x0b, 0x02, 0x20, 0x69, 0xd9, 0xe3, 0x43, 0x4f, 0x24, 0xde, + 0xbe, 0x7d, 0xfb, 0xf6, 0xe1, 0xbd, 0xb7, 0xef, 0x7d, 0x0b, 0xb8, 0xdd, 0x52, 0xcc, 0xa7, 0xbd, + 0xc3, 0x5c, 0x43, 0xeb, 0x6c, 0x34, 0x34, 0xd5, 0x94, 0x15, 0x15, 0xeb, 0xb7, 0x0c, 0x53, 0xd3, + 0xe5, 0x16, 0xbe, 0xa5, 0xa8, 0x26, 0xd6, 0x8f, 0xe4, 0x06, 0xde, 0x30, 0xba, 0xb8, 0xb1, 0xd1, + 0x30, 0x94, 0x5c, 0x57, 0xd7, 0x4c, 0x0d, 0x4d, 0x58, 0x7f, 0x8f, 0xdf, 0xe0, 0xd7, 0x5a, 0x9a, + 0xd6, 0x6a, 0xe3, 0x0d, 0x9b, 0x7a, 0xd8, 0x3b, 0xda, 0x68, 0x62, 0xa3, 0xa1, 0x2b, 0x5d, 0x53, + 0xd3, 0x09, 0x27, 0x7f, 0xd9, 0xcf, 0x61, 0x2a, 0x1d, 0x6c, 0x98, 0x72, 0xa7, 0x4b, 0x19, 0x2e, + 0xf9, 0x19, 0x7e, 0xa0, 0xcb, 0xdd, 0x2e, 0xd6, 0x0d, 0x32, 0x2e, 0x2c, 0xc1, 0xc2, 0x0e, 0x36, + 0xf7, 0xda, 0xbd, 0x96, 0xa2, 0x96, 0xd4, 0x23, 0x4d, 0xc4, 0x9f, 0xf6, 0xb0, 0x61, 0x0a, 0xff, + 0xe0, 0x60, 0xd1, 0x37, 0x60, 0x74, 0x35, 0xd5, 0xc0, 0x08, 0x41, 0x52, 0x95, 0x3b, 0x38, 0xcb, + 0xad, 0x71, 0xeb, 0x53, 0xa2, 0xfd, 0x1f, 0x5d, 0x83, 0xb9, 0x63, 0xac, 0x36, 0x35, 0x5d, 0x3a, + 0xc6, 0xba, 0xa1, 0x68, 0x6a, 0x36, 0x6e, 0x8f, 0xce, 0x12, 0xea, 0x01, 0x21, 0xa2, 0x1d, 0x48, + 0x77, 0x64, 0x55, 0x39, 0xc2, 0x86, 0x99, 0x4d, 0xac, 0x25, 0xd6, 0xa7, 0x37, 0x5f, 0xcf, 0x91, + 0xad, 0xe6, 0x02, 0xd7, 0xca, 0x3d, 0xa4, 0xdc, 0x45, 0xd5, 0xd4, 0x4f, 0x45, 0x77, 0x32, 0x7f, + 0x17, 0x66, 0x99, 0x21, 0x94, 0x81, 0xc4, 0x27, 0xf8, 0x94, 0xea, 0x64, 0xfd, 0x45, 0x0b, 0x90, + 0x3a, 0x96, 0xdb, 0x3d, 0x4c, 0x35, 0x21, 0x0f, 0x77, 0xe2, 0xdf, 0xe6, 0x84, 0x4b, 0xb0, 0xe2, + 0xae, 0x56, 0x90, 0xbb, 0xf2, 0xa1, 0xd2, 0x56, 0x4c, 0x05, 0x1b, 0xce, 0xd6, 0x3f, 0x82, 0xd5, + 0x90, 0x71, 0x6a, 0x81, 0x7b, 0x30, 0xd3, 0xf0, 0xd0, 0xb3, 0x9c, 0xbd, 0x95, 0xac, 0xb3, 0x15, + 0xdf, 0xcc, 0x53, 0x91, 0xe1, 0x16, 0xfe, 0x96, 0x80, 0x8c, 0x9f, 0x05, 0xdd, 0x83, 0x49, 0x03, + 0xeb, 0xc7, 0x4a, 0x83, 0xd8, 0x75, 0x7a, 0x73, 0x2d, 0x4c, 0x5a, 0xae, 0x46, 0xf8, 0x76, 0x63, + 0xa2, 0x33, 0x05, 0xed, 0x43, 0xe6, 0x58, 0x6b, 0xf7, 0x3a, 0x58, 0xc2, 0x27, 0x5d, 0x59, 0x75, + 0x5f, 0xc0, 0xf4, 0xe6, 0x7a, 0xa8, 0x98, 0x03, 0x7b, 0x42, 0xd1, 0xe1, 0xdf, 0x8d, 0x89, 0xe7, + 0x8e, 0x59, 0x12, 0xff, 0x73, 0x0e, 0x26, 0xe9, 0x6a, 0xe8, 0x1d, 0x48, 0x9a, 0xa7, 0x5d, 0xa2, + 0xdd, 0xdc, 0xe6, 0xb5, 0x61, 0xda, 0xe5, 0xea, 0xa7, 0x5d, 0x2c, 0xda, 0x53, 0x84, 0x0f, 0x20, + 0x69, 0x3d, 0xa1, 0x69, 0x98, 0xdc, 0xaf, 0x3c, 0xa8, 0x54, 0x1f, 0x55, 0x32, 0x31, 0xb4, 0x04, + 0xa8, 0x50, 0xad, 0xd4, 0xc5, 0x6a, 0xb9, 0x5c, 0x14, 0xa5, 0x5a, 0x51, 0x3c, 0x28, 0x15, 0x8a, + 0x19, 0x0e, 0xbd, 0x02, 0x6b, 0x07, 0xd5, 0xf2, 0xfe, 0xc3, 0xa2, 0x94, 0x2f, 0x14, 0x8a, 0xb5, + 0x5a, 0x69, 0xab, 0x54, 0x2e, 0xd5, 0x1f, 0x4b, 0x85, 0x6a, 0xa5, 0x56, 0x17, 0xf3, 0xa5, 0x4a, + 0xbd, 0x96, 0x89, 0xf3, 0x3f, 0xe6, 0xe0, 0x9c, 0x6f, 0x03, 0x28, 0xcf, 0x68, 0x78, 0x6b, 0xd4, + 0x8d, 0x7b, 0x35, 0xbd, 0x19, 0xa4, 0x29, 0xc0, 0x44, 0xb5, 0x52, 0x2e, 0x55, 0x2c, 0xed, 0xa6, + 0x61, 0xb2, 0x7a, 0xff, 0xbe, 0xfd, 0x10, 0xdf, 0x9a, 0x20, 0x0b, 0x0a, 0x73, 0x30, 0xb3, 0xa7, + 0x6b, 0x87, 0xd8, 0xf1, 0x9f, 0x3c, 0xcc, 0xd2, 0x67, 0xea, 0x2f, 0xdf, 0x82, 0x94, 0x8e, 0xe5, + 0xe6, 0x29, 0x7d, 0xb5, 0x7c, 0x8e, 0xc4, 0x64, 0xce, 0x89, 0xc9, 0xdc, 0x96, 0xa6, 0xb5, 0x0f, + 0x2c, 0xff, 0x14, 0x09, 0xa3, 0xf0, 0x4d, 0x12, 0xe6, 0x0b, 0x3a, 0x96, 0x4d, 0x4c, 0xb4, 0xa5, + 0xa2, 0x03, 0x63, 0xef, 0x1e, 0xcc, 0x59, 0xfe, 0xd5, 0x50, 0xcc, 0x53, 0x49, 0x97, 0xd5, 0x16, + 0xa6, 0xaf, 0x7e, 0xd1, 0xb1, 0x40, 0x81, 0x8e, 0x8a, 0xd6, 0xa0, 0x38, 0xdb, 0xf0, 0x3e, 0xa2, + 0x12, 0xcc, 0x53, 0xd7, 0x61, 0x5c, 0x3a, 0xc1, 0xba, 0x34, 0xd1, 0xc2, 0xe3, 0xd2, 0xe8, 0x98, + 0xa5, 0x28, 0xd8, 0x40, 0x0f, 0x00, 0xba, 0xb2, 0x2e, 0x77, 0xb0, 0x89, 0x75, 0x23, 0x9b, 0x64, + 0xe3, 0x3b, 0x60, 0x37, 0xb9, 0x3d, 0x97, 0x9b, 0xc4, 0xb7, 0x67, 0x3a, 0xda, 0xb1, 0x02, 0xa2, + 0xa1, 0x63, 0xd3, 0xc8, 0xa6, 0x6c, 0x49, 0xeb, 0x51, 0x92, 0x6a, 0x84, 0xd5, 0x16, 0xb3, 0x95, + 0xf8, 0xc5, 0x16, 0x27, 0x3a, 0xb3, 0x51, 0x15, 0x16, 0x9d, 0x0d, 0x6a, 0xaa, 0x89, 0x55, 0x53, + 0x32, 0xb4, 0x9e, 0xde, 0xc0, 0xd9, 0x09, 0xdb, 0x4a, 0xcb, 0xbe, 0x2d, 0x12, 0x9e, 0x9a, 0xcd, + 0x22, 0x52, 0xd3, 0x30, 0x44, 0xf4, 0x04, 0x78, 0xb9, 0xd1, 0xc0, 0x86, 0xa1, 0x10, 0x5b, 0x48, + 0x3a, 0xfe, 0xb4, 0xa7, 0xe8, 0xb8, 0x83, 0x55, 0xd3, 0xc8, 0x4e, 0xb2, 0x52, 0xeb, 0x5a, 0x57, + 0x6b, 0x6b, 0xad, 0x53, 0xb1, 0xcf, 0x23, 0x5e, 0x64, 0xa6, 0x7b, 0x46, 0x0c, 0xfe, 0x5d, 0x38, + 0xe7, 0x33, 0xca, 0x38, 0x99, 0x8d, 0xbf, 0x03, 0x33, 0x5e, 0x4b, 0x8c, 0x95, 0x15, 0x7f, 0x1a, + 0x87, 0xf9, 0x00, 0x1b, 0xa0, 0x5d, 0x48, 0x1b, 0xaa, 0xdc, 0x35, 0x9e, 0x6a, 0x26, 0xf5, 0xdf, + 0x1b, 0x11, 0x26, 0xcb, 0xd5, 0x28, 0x2f, 0x79, 0xdc, 0x8d, 0x89, 0xee, 0x6c, 0xb4, 0x05, 0x13, + 0xc4, 0x9e, 0xfe, 0xdc, 0x14, 0x24, 0x87, 0xd0, 0x5c, 0x29, 0x74, 0x26, 0xff, 0x06, 0xcc, 0xb1, + 0x2b, 0xa0, 0xcb, 0x30, 0xed, 0xac, 0x20, 0x29, 0x4d, 0xba, 0x57, 0x70, 0x48, 0xa5, 0x26, 0xff, + 0x3a, 0xcc, 0x78, 0x85, 0xa1, 0x65, 0x98, 0xa2, 0x0e, 0xe1, 0xb2, 0xa7, 0x09, 0xa1, 0xd4, 0x74, + 0x63, 0xfa, 0x3b, 0xb0, 0xc0, 0xfa, 0x19, 0x0d, 0xe5, 0xeb, 0xee, 0x1e, 0x88, 0x2d, 0xe6, 0xd8, + 0x3d, 0x38, 0x7a, 0x0a, 0xbf, 0x4b, 0x42, 0xc6, 0x1f, 0x34, 0xe8, 0x1e, 0xa4, 0x0e, 0xdb, 0x5a, + 0xe3, 0x13, 0x3a, 0xf7, 0x95, 0xb0, 0xe8, 0xca, 0x6d, 0x59, 0x5c, 0x84, 0xba, 0x1b, 0x13, 0xc9, + 0x24, 0x6b, 0x76, 0x47, 0xeb, 0xa9, 0x26, 0xb5, 0x5e, 0xf8, 0xec, 0x87, 0x16, 0x57, 0x7f, 0xb6, + 0x3d, 0x09, 0x6d, 0xc3, 0x34, 0x71, 0x3b, 0xa9, 0xa3, 0x35, 0x71, 0x36, 0x61, 0xcb, 0xb8, 0x1a, + 0x2a, 0x23, 0x6f, 0xf3, 0x3e, 0xd4, 0x9a, 0x58, 0x04, 0xd9, 0xfd, 0xcf, 0xcf, 0xc2, 0xb4, 0x47, + 0x37, 0x7e, 0x07, 0xa6, 0x3d, 0x8b, 0xa1, 0x0b, 0x30, 0x79, 0x64, 0x48, 0x6e, 0x12, 0x9e, 0x12, + 0x27, 0x8e, 0x0c, 0x3b, 0x9f, 0x5e, 0x86, 0x69, 0x5b, 0x0b, 0xe9, 0xa8, 0x2d, 0xb7, 0x8c, 0x6c, + 0x7c, 0x2d, 0x61, 0xbd, 0x23, 0x9b, 0x74, 0xdf, 0xa2, 0xf0, 0xff, 0xe2, 0x00, 0xfa, 0x4b, 0xa2, + 0x7b, 0x90, 0xb4, 0xb5, 0x24, 0xa9, 0x7c, 0x7d, 0x04, 0x2d, 0x73, 0xb6, 0xaa, 0xf6, 0x2c, 0xe1, + 0x57, 0x1c, 0x24, 0x6d, 0x31, 0xfe, 0x03, 0xa7, 0x56, 0xaa, 0xec, 0x94, 0x8b, 0x52, 0xa5, 0xba, + 0x5d, 0x94, 0x1e, 0x89, 0xa5, 0x7a, 0x51, 0xcc, 0x70, 0x68, 0x19, 0x2e, 0x78, 0xe9, 0x62, 0x31, + 0xbf, 0x5d, 0x14, 0xa5, 0x6a, 0xa5, 0xfc, 0x38, 0x13, 0x47, 0x3c, 0x2c, 0x3d, 0xdc, 0x2f, 0xd7, + 0x4b, 0x83, 0x63, 0x09, 0xb4, 0x02, 0x59, 0xcf, 0x18, 0x95, 0x41, 0xc5, 0x26, 0x2d, 0xb1, 0x9e, + 0x51, 0xf2, 0x97, 0x0e, 0xa6, 0xb6, 0x66, 0xdd, 0x97, 0x61, 0x3b, 0xdb, 0x23, 0x98, 0x65, 0x72, + 0xb4, 0x55, 0x4e, 0xd1, 0xa4, 0xd2, 0x94, 0x0e, 0x4f, 0x4d, 0xbb, 0xc4, 0xe0, 0xd6, 0x13, 0xe2, + 0xac, 0x43, 0xdd, 0xb2, 0x88, 0x96, 0x59, 0xdb, 0x4a, 0x47, 0x31, 0x29, 0x4f, 0xdc, 0xe6, 0x01, + 0x9b, 0x64, 0x33, 0x08, 0x5f, 0xc5, 0x61, 0x82, 0xbe, 0x9b, 0x6b, 0x9e, 0x53, 0x82, 0x11, 0xe9, + 0x50, 0x89, 0x48, 0x26, 0x38, 0xe2, 0x6c, 0x70, 0xa0, 0x5d, 0x98, 0xf3, 0xa6, 0xd2, 0x13, 0xa7, + 0x88, 0xbb, 0xc2, 0xbe, 0x20, 0x6f, 0x3c, 0x9f, 0xd0, 0xd2, 0x6d, 0xf6, 0xd8, 0x4b, 0x43, 0x5b, + 0x30, 0xe7, 0xcb, 0xc6, 0xc9, 0xe1, 0xd9, 0x78, 0xb6, 0xc1, 0x24, 0xa6, 0x3c, 0xcc, 0x3b, 0x89, + 0xb4, 0x8d, 0x25, 0x93, 0x26, 0x5a, 0x7a, 0x5a, 0x64, 0x06, 0x12, 0x30, 0xea, 0x33, 0x3b, 0x34, + 0xfe, 0x3d, 0x40, 0x83, 0xba, 0x8e, 0x95, 0x35, 0x7b, 0x30, 0x1f, 0x90, 0xe2, 0x51, 0x0e, 0xa6, + 0xec, 0x57, 0x65, 0x28, 0x26, 0xa6, 0xe5, 0xe1, 0xa0, 0x46, 0x7d, 0x16, 0x8b, 0xbf, 0xab, 0xe3, + 0x23, 0xac, 0xeb, 0xb8, 0x69, 0x87, 0x47, 0x20, 0xbf, 0xcb, 0x22, 0xfc, 0x84, 0x83, 0xb4, 0x43, + 0x47, 0x77, 0x20, 0x6d, 0xe0, 0x16, 0x39, 0x7e, 0xc8, 0x5a, 0x97, 0xfc, 0x73, 0x73, 0x35, 0xca, + 0x40, 0x0b, 0x69, 0x87, 0xdf, 0x2a, 0xa4, 0x99, 0xa1, 0xb1, 0x36, 0xff, 0x27, 0x0e, 0xe6, 0xb7, + 0x71, 0x1b, 0xfb, 0xab, 0x94, 0xa8, 0x0c, 0xeb, 0x3d, 0xd8, 0xe3, 0xec, 0xc1, 0x1e, 0x20, 0x2a, + 0xe2, 0x60, 0x3f, 0xd3, 0x61, 0xb7, 0x04, 0x0b, 0xec, 0x6a, 0x24, 0xbd, 0x0b, 0xff, 0x4e, 0xc0, + 0x25, 0xcb, 0x17, 0x74, 0xad, 0xdd, 0xc6, 0xfa, 0x5e, 0xef, 0xb0, 0xad, 0x18, 0x4f, 0xc7, 0xd8, + 0xdc, 0x05, 0x98, 0x54, 0xb5, 0xa6, 0x27, 0x78, 0x26, 0xac, 0xc7, 0x52, 0x13, 0x15, 0xe1, 0xbc, + 0xbf, 0xcc, 0x3a, 0xa5, 0x49, 0x38, 0xbc, 0xc8, 0xca, 0x1c, 0xfb, 0x4f, 0x10, 0x1e, 0xd2, 0x56, + 0x81, 0xa8, 0xa9, 0xed, 0x53, 0x3b, 0x62, 0xd2, 0xa2, 0xfb, 0x8c, 0x44, 0x7f, 0xc5, 0xf4, 0xa6, + 0x5b, 0x31, 0x45, 0xee, 0x28, 0xaa, 0x78, 0xfa, 0x78, 0x20, 0xe2, 0x27, 0x6c, 0xd1, 0xef, 0x8c, + 0x28, 0x7a, 0x68, 0x26, 0x38, 0xcb, 0x5b, 0x7c, 0x0e, 0xe1, 0xfb, 0x57, 0x0e, 0x2e, 0x87, 0x6e, + 0x81, 0x1e, 0xf9, 0x4d, 0x38, 0xd7, 0x25, 0x03, 0xae, 0x11, 0x48, 0x94, 0xdd, 0x1d, 0x6a, 0x04, + 0xda, 0xc5, 0x52, 0x2a, 0x63, 0x86, 0xb9, 0x2e, 0x43, 0xe4, 0xf3, 0x30, 0x1f, 0xc0, 0x36, 0xd6, + 0x66, 0xbe, 0xe6, 0x60, 0xad, 0xaf, 0xca, 0xbe, 0xda, 0x7d, 0x7e, 0xee, 0x5b, 0xef, 0xfb, 0x16, + 0x49, 0xf9, 0xb7, 0x07, 0xf7, 0x1e, 0xbc, 0xe0, 0x8b, 0x8a, 0xe0, 0xab, 0x70, 0x25, 0x62, 0x69, + 0x1a, 0xce, 0x5f, 0x25, 0xe1, 0xca, 0x81, 0xdc, 0x56, 0x9a, 0x6e, 0x21, 0x17, 0xd0, 0xef, 0x47, + 0x9b, 0xa4, 0x31, 0x10, 0x01, 0x24, 0x6b, 0xdd, 0x73, 0xa3, 0x76, 0x98, 0xfc, 0x11, 0x8e, 0xc3, + 0xe7, 0xd8, 0x84, 0x3d, 0x0e, 0x68, 0xc2, 0xde, 0x19, 0x5d, 0xd7, 0xa8, 0x96, 0x6c, 0xdf, 0x9f, + 0x60, 0xde, 0x1e, 0x5d, 0x6e, 0x84, 0x17, 0x9c, 0x39, 0x8a, 0x5f, 0x66, 0xd7, 0xf4, 0x97, 0x24, + 0x08, 0x51, 0xbb, 0xa7, 0x39, 0x44, 0x84, 0xa9, 0x86, 0xa6, 0x1e, 0x29, 0x7a, 0x07, 0x37, 0x69, + 0xf5, 0xff, 0xd6, 0x28, 0xc6, 0xa3, 0x09, 0xa4, 0xe0, 0xcc, 0x15, 0xfb, 0x62, 0x50, 0x16, 0x26, + 0x3b, 0xd8, 0x30, 0xe4, 0x96, 0xa3, 0x96, 0xf3, 0xc8, 0x7f, 0x91, 0x80, 0x29, 0x77, 0x0a, 0x52, + 0x07, 0x3c, 0x98, 0xa4, 0xaf, 0x9d, 0x67, 0x51, 0xe0, 0xd9, 0x9d, 0x39, 0xfe, 0x0c, 0xce, 0xdc, + 0x64, 0x9c, 0x99, 0x84, 0xc3, 0xf6, 0x33, 0xa9, 0x1d, 0xe1, 0xd7, 0x2f, 0xdd, 0x01, 0x85, 0xef, + 0x03, 0x2a, 0x2b, 0x06, 0xed, 0xa2, 0xdc, 0xb4, 0x64, 0x35, 0x4d, 0xf2, 0x89, 0x84, 0x55, 0x53, + 0x57, 0x68, 0xb9, 0x9e, 0x12, 0xa1, 0x23, 0x9f, 0x14, 0x09, 0xc5, 0x2a, 0xe9, 0x0d, 0x53, 0xd6, + 0x4d, 0x45, 0x6d, 0x49, 0xa6, 0xf6, 0x09, 0x76, 0x41, 0x57, 0x87, 0x5a, 0xb7, 0x88, 0xc2, 0xe7, + 0x1c, 0xcc, 0x33, 0xe2, 0xa9, 0x4f, 0xde, 0x85, 0xc9, 0xbe, 0x6c, 0xa6, 0x8c, 0x0f, 0xe0, 0xce, + 0x11, 0xb3, 0x39, 0x33, 0xd0, 0x2a, 0x80, 0x8a, 0x4f, 0x4c, 0x66, 0xdd, 0x29, 0x8b, 0x62, 0xaf, + 0xc9, 0x6f, 0x40, 0x8a, 0x98, 0x61, 0xd4, 0x7e, 0xf9, 0x8b, 0x38, 0xa0, 0x1d, 0x6c, 0xba, 0x6d, + 0x10, 0xb5, 0x41, 0x88, 0x2f, 0x71, 0xcf, 0xe0, 0x4b, 0xef, 0x33, 0xbe, 0x44, 0xbc, 0xf1, 0x86, + 0x07, 0x7d, 0xf6, 0x2d, 0x1d, 0x99, 0x09, 0x43, 0x5a, 0x0f, 0x52, 0xcf, 0x8d, 0xd6, 0x7a, 0x9c, + 0xd1, 0x65, 0xb6, 0x61, 0x9e, 0xd1, 0x99, 0xbe, 0xd3, 0x5b, 0x80, 0xe4, 0x63, 0x59, 0x69, 0xcb, + 0x96, 0x5e, 0x4e, 0x67, 0x47, 0x3b, 0xbd, 0xf3, 0xee, 0x88, 0x33, 0x4d, 0x10, 0xbc, 0x05, 0x03, + 0x95, 0xe7, 0x47, 0xc3, 0xdb, 0xde, 0x83, 0x76, 0x80, 0x87, 0xae, 0xbb, 0x13, 0x88, 0x88, 0x5f, + 0x1d, 0x2c, 0x12, 0x28, 0x3c, 0x1c, 0x0a, 0x8e, 0x7f, 0x96, 0x80, 0xe5, 0x08, 0x6e, 0x74, 0x17, + 0x12, 0x7a, 0xb7, 0x41, 0x9d, 0xe9, 0xd5, 0x11, 0xe4, 0xe7, 0xc4, 0xbd, 0xc2, 0x6e, 0x4c, 0xb4, + 0x66, 0xf1, 0xbf, 0x89, 0x43, 0x42, 0xdc, 0x2b, 0xa0, 0xf7, 0x18, 0xa4, 0xf8, 0xe6, 0x88, 0x52, + 0xbc, 0x40, 0xf1, 0x97, 0x5c, 0x10, 0x52, 0x9c, 0x85, 0x85, 0x82, 0x58, 0xcc, 0xd7, 0x8b, 0xd2, + 0x76, 0xb1, 0x5c, 0xac, 0x17, 0x25, 0x82, 0x64, 0x67, 0x38, 0xb4, 0x02, 0xd9, 0xbd, 0xfd, 0xad, + 0x72, 0xa9, 0xb6, 0x2b, 0xed, 0x57, 0x9c, 0x7f, 0x74, 0x34, 0x8e, 0x32, 0x30, 0x53, 0x2e, 0xd5, + 0xea, 0x94, 0x50, 0xcb, 0x24, 0x2c, 0xca, 0x4e, 0xb1, 0x2e, 0x15, 0xf2, 0x7b, 0xf9, 0x42, 0xa9, + 0xfe, 0x38, 0x93, 0x44, 0x3c, 0x2c, 0xb1, 0xb2, 0x6b, 0x95, 0xfc, 0x5e, 0x6d, 0xb7, 0x5a, 0xcf, + 0xa4, 0x10, 0x82, 0x39, 0x7b, 0xbe, 0x43, 0xaa, 0x65, 0x26, 0x2c, 0x09, 0x85, 0x72, 0xb5, 0xe2, + 0xea, 0x30, 0x89, 0x16, 0x20, 0xe3, 0xac, 0x2c, 0x16, 0xf3, 0xdb, 0x36, 0x8a, 0x91, 0x46, 0xe7, + 0x61, 0xb6, 0xf8, 0xe1, 0x5e, 0xbe, 0xb2, 0xed, 0x30, 0x4e, 0xb9, 0x18, 0xd8, 0xd7, 0x71, 0x58, + 0x24, 0x20, 0x98, 0x03, 0xb9, 0x39, 0x61, 0xb9, 0x0e, 0x19, 0xd2, 0xb6, 0x4b, 0xfe, 0xc2, 0x69, + 0x8e, 0xd0, 0x0f, 0x9c, 0xf2, 0xc9, 0x01, 0xac, 0xe3, 0x1e, 0xc0, 0xba, 0xe4, 0x2f, 0x26, 0x6f, + 0xb0, 0xd0, 0xae, 0x6f, 0xb5, 0xa8, 0xfe, 0xe4, 0x61, 0x40, 0xb5, 0x73, 0x2b, 0x5a, 0x5a, 0xd4, + 0x49, 0x70, 0x96, 0x66, 0xe4, 0x8c, 0x01, 0x7d, 0x1f, 0x96, 0xfc, 0xfa, 0xd2, 0xd8, 0xba, 0x39, + 0x00, 0xc0, 0xba, 0x19, 0xc6, 0xe5, 0x75, 0x39, 0x84, 0xbf, 0x73, 0x90, 0x76, 0xc8, 0x56, 0x96, + 0x36, 0x94, 0x1f, 0x62, 0x06, 0xf0, 0x99, 0xb2, 0x28, 0x2e, 0x7e, 0xe4, 0x85, 0x4e, 0xe3, 0x7e, + 0xe8, 0x34, 0xf0, 0x3d, 0x27, 0x02, 0xdf, 0xf3, 0x77, 0x61, 0xb6, 0x61, 0xa9, 0xaf, 0x68, 0xaa, + 0x64, 0x2a, 0x1d, 0x07, 0xcf, 0x19, 0xbc, 0xea, 0xa8, 0x3b, 0xf7, 0x93, 0xe2, 0x8c, 0x33, 0xc1, + 0x22, 0xa1, 0x35, 0x98, 0xb1, 0xaf, 0x3e, 0x24, 0x53, 0x93, 0x7a, 0x06, 0xce, 0xa6, 0xec, 0xee, + 0x16, 0x6c, 0x5a, 0x5d, 0xdb, 0x37, 0xb0, 0xf0, 0x67, 0x0e, 0x16, 0x49, 0xd3, 0xee, 0x77, 0xc7, + 0x61, 0x10, 0xb0, 0xd7, 0xe3, 0x7c, 0x89, 0x3f, 0x50, 0xe0, 0x8b, 0xea, 0x59, 0xb2, 0xb0, 0xe4, + 0x5f, 0x8f, 0x36, 0x2a, 0xbf, 0xe5, 0x60, 0xc1, 0x3a, 0x75, 0x9d, 0x81, 0xe7, 0x5d, 0x04, 0x8c, + 0xf1, 0x26, 0x7d, 0xc6, 0x4c, 0xfa, 0x8d, 0x29, 0xfc, 0x9e, 0x83, 0x45, 0x9f, 0xae, 0xd4, 0x53, + 0xdf, 0xf5, 0x57, 0x14, 0x57, 0xbd, 0x15, 0xc5, 0x00, 0xff, 0x98, 0x35, 0xc5, 0x6d, 0xa7, 0xa6, + 0x18, 0x2f, 0x20, 0x3e, 0x8b, 0xc3, 0x6a, 0x3f, 0xb3, 0xdb, 0xd7, 0x7e, 0xcd, 0x31, 0x5a, 0xe2, + 0xb3, 0xdd, 0xae, 0x7d, 0xe0, 0x4f, 0x75, 0x9b, 0x83, 0x87, 0x4d, 0x80, 0x4a, 0x2f, 0xca, 0x01, + 0x7f, 0xe4, 0x45, 0xb7, 0xd8, 0x75, 0xe9, 0x2b, 0x1c, 0x11, 0x26, 0x7e, 0x1b, 0x2e, 0xd8, 0x40, + 0x81, 0x7b, 0xdd, 0xec, 0x5c, 0x82, 0x91, 0x2c, 0x92, 0x16, 0x17, 0xad, 0x61, 0xf7, 0x8e, 0x95, + 0x42, 0xa3, 0x4d, 0xe1, 0x9b, 0x24, 0x2c, 0x55, 0xb4, 0x26, 0xae, 0x99, 0x72, 0x6b, 0x1c, 0xd0, + 0xf0, 0x7b, 0x83, 0x18, 0x4c, 0x9c, 0xb5, 0x67, 0xb0, 0xd4, 0x51, 0xa0, 0x17, 0x94, 0x83, 0x79, + 0xc3, 0x94, 0x5b, 0x76, 0x04, 0xc9, 0x7a, 0x0b, 0x9b, 0x52, 0x57, 0x36, 0x9f, 0xd2, 0xf0, 0x38, + 0x4f, 0x87, 0xea, 0xf6, 0xc8, 0x9e, 0x6c, 0x3e, 0x0d, 0xc6, 0xf2, 0x92, 0x63, 0x63, 0x79, 0xef, + 0xfb, 0xdb, 0xe9, 0xd7, 0x87, 0xec, 0x25, 0xe2, 0x1c, 0xfc, 0x30, 0x04, 0xa7, 0x7b, 0x63, 0x88, + 0xc8, 0xe1, 0xf8, 0xdc, 0xd9, 0x71, 0xa9, 0x97, 0x0c, 0xf1, 0x5d, 0x84, 0x0b, 0x03, 0x9b, 0xa7, + 0x59, 0xb7, 0x05, 0x59, 0x6b, 0x68, 0x5f, 0x35, 0xc6, 0x74, 0xc7, 0x10, 0x8f, 0x89, 0x87, 0x78, + 0x8c, 0xb0, 0x0c, 0x17, 0x03, 0x16, 0xa2, 0x5a, 0xfc, 0x31, 0x45, 0xd4, 0x18, 0x1f, 0x6d, 0xfe, + 0x28, 0x2c, 0x2a, 0xde, 0xf2, 0xbe, 0xf6, 0x40, 0x60, 0xf6, 0x45, 0xc4, 0xc5, 0x65, 0x98, 0xf6, + 0xf2, 0xd1, 0x93, 0xc3, 0x1c, 0x12, 0x38, 0xa9, 0x33, 0x81, 0xe0, 0x13, 0x3e, 0x10, 0xbc, 0xdc, + 0x0f, 0xaa, 0x49, 0xb6, 0x1a, 0x0c, 0x35, 0x45, 0x44, 0x58, 0x3d, 0x19, 0x08, 0xab, 0x34, 0x8b, + 0xac, 0x87, 0x0a, 0xfd, 0x3f, 0x08, 0x2c, 0xea, 0xd4, 0x81, 0x90, 0xb7, 0xf0, 0x04, 0x78, 0xe2, + 0xf1, 0xe3, 0x83, 0xd0, 0x3e, 0x37, 0x8a, 0xfb, 0xdd, 0x48, 0x58, 0x85, 0xe5, 0x40, 0xd9, 0x74, + 0xe9, 0xc7, 0x44, 0xaf, 0x1d, 0x4c, 0x31, 0x8c, 0x9a, 0x29, 0x9b, 0xc6, 0xa8, 0x2b, 0xd3, 0x41, + 0xef, 0xca, 0x84, 0x64, 0xaf, 0xbc, 0x43, 0x76, 0xe5, 0x17, 0x4d, 0xcf, 0xce, 0xd7, 0x20, 0xd5, + 0xb3, 0xe1, 0x38, 0x52, 0xfc, 0xcc, 0xb3, 0x2e, 0xbd, 0x6f, 0x0d, 0x89, 0x84, 0x43, 0xf8, 0x03, + 0x07, 0xd3, 0x1e, 0x32, 0x5a, 0x81, 0x29, 0xb7, 0x3b, 0x77, 0xea, 0x74, 0x97, 0x60, 0xbd, 0x03, + 0x53, 0x33, 0xe5, 0x36, 0xbd, 0xe1, 0x25, 0x0f, 0x56, 0x6b, 0xd5, 0x33, 0x30, 0x29, 0xe3, 0x12, + 0xa2, 0xfd, 0x1f, 0xdd, 0x84, 0x64, 0x4f, 0x55, 0x4c, 0x3b, 0xf6, 0xe6, 0xfc, 0x41, 0x65, 0x2f, + 0x95, 0xdb, 0x57, 0x15, 0x53, 0xb4, 0xb9, 0x84, 0x1b, 0x90, 0xb4, 0x9e, 0xd8, 0x26, 0x76, 0x0a, + 0x52, 0x5b, 0x8f, 0xeb, 0xc5, 0x5a, 0x86, 0x43, 0x00, 0x13, 0xa5, 0x4a, 0x75, 0xbb, 0x58, 0xcb, + 0xc4, 0x85, 0x15, 0x77, 0xeb, 0x41, 0x20, 0xc1, 0xc7, 0xe4, 0x95, 0x84, 0xc1, 0x03, 0xf9, 0x40, + 0x78, 0x60, 0x95, 0x39, 0x9c, 0x86, 0x00, 0x03, 0xff, 0xe1, 0x60, 0x31, 0x90, 0x0f, 0xdd, 0xf6, + 0x42, 0x02, 0x57, 0x22, 0x65, 0x7a, 0xc1, 0x80, 0x5f, 0x72, 0x04, 0x0c, 0xb8, 0xc3, 0x80, 0x01, + 0xd7, 0x87, 0xce, 0xf7, 0xc2, 0x00, 0x07, 0x21, 0x28, 0x40, 0xad, 0x9e, 0xdf, 0x29, 0x4a, 0xfb, + 0x15, 0xf2, 0xeb, 0xa2, 0x00, 0x0b, 0x90, 0xb1, 0xba, 0x7a, 0xfa, 0x7d, 0x5b, 0xad, 0x9e, 0xaf, + 0xd7, 0x32, 0xf1, 0xc1, 0x0e, 0x3c, 0xe1, 0x76, 0xe0, 0x0b, 0x80, 0xa8, 0x59, 0xbd, 0x9f, 0x66, + 0x7e, 0xce, 0xc1, 0x3c, 0x43, 0xa6, 0x56, 0xf6, 0xdc, 0xde, 0x70, 0xcc, 0xed, 0xcd, 0x06, 0x2c, + 0x58, 0x4d, 0x04, 0x71, 0x64, 0x43, 0xea, 0x62, 0x5d, 0xb2, 0x46, 0xa8, 0x3b, 0x9d, 0xef, 0xc8, + 0x27, 0x14, 0xed, 0xdb, 0xc3, 0xba, 0x25, 0xf8, 0x39, 0xe0, 0x5b, 0xc2, 0xcf, 0x38, 0x72, 0xee, + 0x8e, 0x5d, 0x70, 0x0f, 0x0b, 0xc2, 0x80, 0x8a, 0x3c, 0x31, 0x7a, 0x45, 0x2e, 0xe4, 0xc9, 0x61, + 0x7b, 0x86, 0xe2, 0x77, 0xf3, 0xbf, 0x1c, 0xa4, 0x4b, 0x4d, 0xac, 0x9a, 0x96, 0xf7, 0x55, 0x60, + 0x96, 0xf9, 0x74, 0x15, 0xad, 0x84, 0x7c, 0xd1, 0x6a, 0x6f, 0x9d, 0x5f, 0x8d, 0xfc, 0xde, 0x55, + 0x88, 0xa1, 0x23, 0xcf, 0x67, 0xb7, 0x0c, 0x7e, 0xf9, 0xca, 0xc0, 0xcc, 0x80, 0x40, 0xe4, 0xaf, + 0x0d, 0xe1, 0x72, 0xd7, 0x79, 0x1b, 0x52, 0xf6, 0x47, 0x8a, 0x68, 0xc1, 0xfd, 0x50, 0xd2, 0xf3, + 0x0d, 0x23, 0xbf, 0xe8, 0xa3, 0x3a, 0xf3, 0x36, 0xff, 0x99, 0x06, 0xe8, 0xf7, 0x10, 0xe8, 0x01, + 0xcc, 0x78, 0xbf, 0x93, 0x42, 0xcb, 0x11, 0x5f, 0xe9, 0xf1, 0x2b, 0xc1, 0x83, 0xae, 0x4e, 0x0f, + 0x60, 0xc6, 0x7b, 0x2b, 0xdf, 0x17, 0x16, 0xf0, 0x65, 0x40, 0x5f, 0x58, 0xe0, 0x45, 0x7e, 0x0c, + 0xb5, 0xe1, 0x42, 0xc8, 0xbd, 0x2c, 0xba, 0x3e, 0xda, 0xed, 0x35, 0xff, 0xea, 0x88, 0x17, 0xbc, + 0x42, 0x0c, 0xe9, 0x70, 0x31, 0xf4, 0x3a, 0x12, 0xad, 0x8f, 0x7a, 0x59, 0xca, 0xbf, 0x36, 0x02, + 0xa7, 0xbb, 0x66, 0x0f, 0xf8, 0xf0, 0x3b, 0x10, 0xf4, 0xda, 0xc8, 0x97, 0x73, 0xfc, 0x8d, 0xd1, + 0xaf, 0x54, 0x84, 0x18, 0xda, 0x85, 0x69, 0xcf, 0x05, 0x01, 0xe2, 0x03, 0x6f, 0x0d, 0x88, 0xe0, + 0xe5, 0x88, 0x1b, 0x05, 0x22, 0xc9, 0x03, 0x62, 0xf7, 0x25, 0x0d, 0xa2, 0xf1, 0x7d, 0x49, 0x01, + 0xa8, 0xb7, 0xdf, 0xfc, 0xbe, 0x53, 0x28, 0xc8, 0xfc, 0xc1, 0xc7, 0x58, 0x90, 0xf9, 0x43, 0x8e, + 0x34, 0x21, 0x86, 0x3e, 0x80, 0x39, 0x16, 0xb1, 0x43, 0xab, 0x91, 0xc8, 0x23, 0x7f, 0x29, 0x6c, + 0xd8, 0x2b, 0x92, 0x05, 0x88, 0xfa, 0x22, 0x03, 0x81, 0xaa, 0xbe, 0xc8, 0x10, 0x5c, 0x29, 0x66, + 0xe5, 0x27, 0x06, 0x7c, 0xe9, 0xe7, 0xa7, 0x20, 0xbc, 0xa9, 0x9f, 0x9f, 0x02, 0x11, 0x1b, 0x21, + 0x86, 0x14, 0x58, 0x0a, 0x86, 0x10, 0xd0, 0xb5, 0x91, 0xa0, 0x0d, 0xfe, 0xfa, 0x30, 0x36, 0x37, + 0xd5, 0x7c, 0x99, 0x82, 0xa4, 0x7d, 0x1c, 0xd5, 0xe1, 0x9c, 0xaf, 0x85, 0x43, 0x97, 0xa2, 0x1b, + 0x5b, 0xfe, 0x72, 0xe8, 0xb8, 0xbb, 0x93, 0x27, 0x70, 0x7e, 0xa0, 0x29, 0x43, 0x6b, 0xde, 0x79, + 0x41, 0x8d, 0x21, 0x7f, 0x25, 0x82, 0xc3, 0x2f, 0x9b, 0x4d, 0x3b, 0x6b, 0xc3, 0xba, 0x06, 0x56, + 0x76, 0x58, 0xaa, 0xf9, 0x98, 0x9c, 0xfe, 0xfe, 0x24, 0x23, 0xb0, 0x7a, 0x05, 0xa6, 0x97, 0xab, + 0x91, 0x3c, 0xee, 0x0a, 0x1f, 0xb9, 0x65, 0x87, 0xa7, 0xcc, 0x45, 0x8c, 0x72, 0x81, 0xd5, 0x35, + 0x2f, 0x44, 0xb1, 0xb8, 0xe2, 0x1f, 0x41, 0xc6, 0x7f, 0x04, 0x23, 0xe6, 0x7d, 0x05, 0xb9, 0xcd, + 0x5a, 0x38, 0x83, 0xdf, 0x32, 0xfe, 0xf8, 0xf7, 0x6b, 0x15, 0x14, 0xf9, 0x57, 0x23, 0x79, 0xbc, + 0x19, 0xcb, 0x53, 0x79, 0xf5, 0x33, 0xd6, 0x60, 0x95, 0xd6, 0xcf, 0x58, 0x01, 0xa5, 0x9a, 0x10, + 0xbb, 0xf3, 0x2e, 0x40, 0xc3, 0x50, 0x24, 0xd2, 0x69, 0xa2, 0xd5, 0x01, 0x9c, 0xfc, 0xbe, 0x82, + 0xdb, 0xcd, 0x6a, 0xd7, 0x54, 0x34, 0xd5, 0xc8, 0xfe, 0x3a, 0x6d, 0xb7, 0xb9, 0x53, 0x0d, 0x43, + 0x21, 0x0d, 0xdf, 0x56, 0xea, 0x49, 0xa2, 0x61, 0x28, 0x87, 0x13, 0x36, 0xff, 0x9b, 0xff, 0x0b, + 0x00, 0x00, 0xff, 0xff, 0x95, 0x8e, 0xb8, 0x49, 0x68, 0x34, 0x00, 0x00, }