From 34dc6b2587050588bfb00e06456b58729d823170 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 10 Mar 2022 11:29:45 +0100 Subject: [PATCH 01/22] Add SELinuxMountReadWriteOncePod feature gate --- pkg/features/kube_features.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 426f28b758a..884391f6353 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -860,6 +860,14 @@ const ( // Allow users to specify whether to take nodeAffinity/nodeTaint into consideration when // calculating pod topology spread skew. NodeInclusionPolicyInPodTopologySpread featuregate.Feature = "NodeInclusionPolicyInPodTopologySpread" + + // owner: @jsafrane + // kep: http://kep.k8s.io/1710 + // alpha: v1.25 + // Speed up container startup by mounting volumes with the correct SELinux label + // instead of changing each file on the volumes recursively. + // Initial implementation focused on ReadWriteOncePod volumes. + SELinuxMountReadWriteOncePod featuregate.Feature = "SELinuxMountReadWriteOncePod" ) func init() { @@ -1099,6 +1107,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NodeInclusionPolicyInPodTopologySpread: {Default: false, PreRelease: featuregate.Alpha}, + SELinuxMountReadWriteOncePod: {Default: false, PreRelease: featuregate.Alpha}, + // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: From 3efeeef346f01e688051f83c601baf031c3bc8f5 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 16 Mar 2022 16:20:44 +0100 Subject: [PATCH 02/22] Add CSIDriverSpec.SELinuxMount The new field tells Kubernetes if the CSI driver supports mounting of volumes with -o context=XYZ or not. --- pkg/apis/storage/types.go | 21 +++++ pkg/apis/storage/v1/defaults.go | 4 + pkg/apis/storage/v1/defaults_test.go | 27 ++++++ pkg/apis/storage/v1beta1/defaults.go | 4 + pkg/apis/storage/v1beta1/defaults_test.go | 27 ++++++ pkg/registry/storage/csidriver/strategy.go | 8 ++ .../storage/csidriver/strategy_test.go | 85 ++++++++++++++++--- staging/src/k8s.io/api/storage/v1/types.go | 21 +++++ .../src/k8s.io/api/storage/v1beta1/types.go | 21 +++++ 9 files changed, 206 insertions(+), 12 deletions(-) diff --git a/pkg/apis/storage/types.go b/pkg/apis/storage/types.go index 6c6fd11d804..bde08724a71 100644 --- a/pkg/apis/storage/types.go +++ b/pkg/apis/storage/types.go @@ -390,6 +390,27 @@ type CSIDriverSpec struct { // // +optional RequiresRepublish *bool + + // SELinuxMount specifies if the CSI driver supports "-o context" + // mount option. + // + // When "true", the CSI driver must ensure that all volumes provided by this CSI + // driver can be mounted separately with different `-o context` options. This is + // typical for storage backends that provide volumes as filesystems on block + // devices or as independent shared volumes. + // Kubernetes will call NodeStage / NodePublish with "-o context=xyz" mount + // option when mounting a ReadWriteOncePod volume used in Pod that has + // explicitly set SELinux context. In the future, it may be expanded to other + // volume AccessModes. In any case, Kubernetes will ensure that the volume is + // mounted only with a single SELinux context. + // + // When "false", Kubernetes won't pass any special SELinux mount options to the driver. + // This is typical for volumes that represent subdirectories of a bigger shared filesystem. + // + // Default is "false". + // + // +optional + SELinuxMount *bool } // FSGroupPolicy specifies if a CSI Driver supports modifying diff --git a/pkg/apis/storage/v1/defaults.go b/pkg/apis/storage/v1/defaults.go index 5a6d40dd229..1f2dfc0f2ee 100644 --- a/pkg/apis/storage/v1/defaults.go +++ b/pkg/apis/storage/v1/defaults.go @@ -64,4 +64,8 @@ func SetDefaults_CSIDriver(obj *storagev1.CSIDriver) { obj.Spec.RequiresRepublish = new(bool) *(obj.Spec.RequiresRepublish) = false } + if obj.Spec.SELinuxMount == nil && utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + obj.Spec.SELinuxMount = new(bool) + *(obj.Spec.SELinuxMount) = false + } } diff --git a/pkg/apis/storage/v1/defaults_test.go b/pkg/apis/storage/v1/defaults_test.go index 3c482e086b7..0d5b744c77f 100644 --- a/pkg/apis/storage/v1/defaults_test.go +++ b/pkg/apis/storage/v1/defaults_test.go @@ -122,3 +122,30 @@ func TestSetDefaultCSIDriver(t *testing.T) { }) } } + +func TestSetDefaultSELinuxMountReadWriteOncePodEnabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + driver := &storagev1.CSIDriver{} + + // field should be defaulted + defaultSELinuxMount := false + output := roundTrip(t, runtime.Object(driver)).(*storagev1.CSIDriver) + outSELinuxMount := output.Spec.SELinuxMount + if outSELinuxMount == nil { + t.Errorf("Expected SELinuxMount to be defaulted to: %+v, got: nil", defaultSELinuxMount) + } else if *outSELinuxMount != defaultSELinuxMount { + t.Errorf("Expected SELinuxMount to be defaulted to: %+v, got: %+v", defaultSELinuxMount, outSELinuxMount) + } +} + +func TestSetDefaultSELinuxMountReadWriteOncePodDisabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, false)() + driver := &storagev1.CSIDriver{} + + // field should not be defaulted + output := roundTrip(t, runtime.Object(driver)).(*storagev1.CSIDriver) + outSELinuxMount := output.Spec.SELinuxMount + if outSELinuxMount != nil { + t.Errorf("Expected SELinuxMount to remain nil, got: %+v", outSELinuxMount) + } +} diff --git a/pkg/apis/storage/v1beta1/defaults.go b/pkg/apis/storage/v1beta1/defaults.go index c6d853203af..7897a29f1a5 100644 --- a/pkg/apis/storage/v1beta1/defaults.go +++ b/pkg/apis/storage/v1beta1/defaults.go @@ -64,4 +64,8 @@ func SetDefaults_CSIDriver(obj *storagev1beta1.CSIDriver) { obj.Spec.RequiresRepublish = new(bool) *(obj.Spec.RequiresRepublish) = false } + if obj.Spec.SELinuxMount == nil && utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + obj.Spec.SELinuxMount = new(bool) + *(obj.Spec.SELinuxMount) = false + } } diff --git a/pkg/apis/storage/v1beta1/defaults_test.go b/pkg/apis/storage/v1beta1/defaults_test.go index 317c8a78a96..3dcb0229373 100644 --- a/pkg/apis/storage/v1beta1/defaults_test.go +++ b/pkg/apis/storage/v1beta1/defaults_test.go @@ -165,3 +165,30 @@ func TestSetDefaultCSIDriver(t *testing.T) { }) } } + +func TestSetDefaultSELinuxMountReadWriteOncePodEnabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + driver := &storagev1beta1.CSIDriver{} + + // field should be defaulted + defaultSELinuxMount := false + output := roundTrip(t, runtime.Object(driver)).(*storagev1beta1.CSIDriver) + outSELinuxMount := output.Spec.SELinuxMount + if outSELinuxMount == nil { + t.Errorf("Expected SELinuxMount to be defaulted to: %+v, got: nil", defaultSELinuxMount) + } else if *outSELinuxMount != defaultSELinuxMount { + t.Errorf("Expected SELinuxMount to be defaulted to: %+v, got: %+v", defaultSELinuxMount, outSELinuxMount) + } +} + +func TestSetDefaultSELinuxMountReadWriteOncePodDisabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, false)() + driver := &storagev1beta1.CSIDriver{} + + // field should not be defaulted + output := roundTrip(t, runtime.Object(driver)).(*storagev1beta1.CSIDriver) + outSELinuxMount := output.Spec.SELinuxMount + if outSELinuxMount != nil { + t.Errorf("Expected SELinuxMount remain nil, got: %+v", outSELinuxMount) + } +} diff --git a/pkg/registry/storage/csidriver/strategy.go b/pkg/registry/storage/csidriver/strategy.go index d068cefdc5e..d266dfa0431 100644 --- a/pkg/registry/storage/csidriver/strategy.go +++ b/pkg/registry/storage/csidriver/strategy.go @@ -50,6 +50,9 @@ func (csiDriverStrategy) PrepareForCreate(ctx context.Context, obj runtime.Objec if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { csiDriver.Spec.VolumeLifecycleModes = nil } + if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + csiDriver.Spec.SELinuxMount = nil + } } func (csiDriverStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -87,6 +90,11 @@ func (csiDriverStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime. if !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.TokenRequests, newCSIDriver.Spec.TokenRequests) || !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.RequiresRepublish, newCSIDriver.Spec.RequiresRepublish) { newCSIDriver.Generation = oldCSIDriver.Generation + 1 } + + if oldCSIDriver.Spec.SELinuxMount == nil && + !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + newCSIDriver.Spec.SELinuxMount = nil + } } func (csiDriverStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/storage/csidriver/strategy_test.go b/pkg/registry/storage/csidriver/strategy_test.go index 59c25bdb92f..e0172823188 100644 --- a/pkg/registry/storage/csidriver/strategy_test.go +++ b/pkg/registry/storage/csidriver/strategy_test.go @@ -211,18 +211,36 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { RequiresRepublish: &enabled, }, } + driverWithSELinuxMountEnabled := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + SELinuxMount: &enabled, + }, + } + driverWithSELinuxMountDisabled := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + SELinuxMount: &disabled, + }, + } resultPersistent := []storage.VolumeLifecycleMode{storage.VolumeLifecyclePersistent} tests := []struct { - name string - old, update *storage.CSIDriver - csiInlineVolumeEnabled bool - wantCapacity *bool - wantModes []storage.VolumeLifecycleMode - wantTokenRequests []storage.TokenRequest - wantRequiresRepublish *bool - wantGeneration int64 + name string + old, update *storage.CSIDriver + csiInlineVolumeEnabled bool + seLinuxMountReadWriteOncePodEnabled bool + wantCapacity *bool + wantModes []storage.VolumeLifecycleMode + wantTokenRequests []storage.TokenRequest + wantRequiresRepublish *bool + wantGeneration int64 + wantSELinuxMount *bool }{ { name: "capacity feature enabled, before: none, update: enabled", @@ -237,20 +255,20 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { wantCapacity: &disabled, }, { - name: "inline feature enabled, before: none, update: persitent", + name: "inline feature enabled, before: none, update: persistent", csiInlineVolumeEnabled: true, old: driverWithNothing, update: driverWithPersistent, wantModes: resultPersistent, }, { - name: "inline feature disabled, before: none, update: persitent", + name: "inline feature disabled, before: none, update: persistent", old: driverWithNothing, update: driverWithPersistent, wantModes: nil, }, { - name: "inline feature disabled, before: ephemeral, update: persitent", + name: "inline feature disabled, before: ephemeral, update: persistent", old: driverWithEphemeral, update: driverWithPersistent, wantModes: resultPersistent, @@ -263,11 +281,54 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { wantRequiresRepublish: &enabled, wantGeneration: 1, }, + { + name: "SELinux mount support feature enabled, before: nil, update: on", + seLinuxMountReadWriteOncePodEnabled: true, + old: driverWithNothing, + update: driverWithSELinuxMountEnabled, + wantSELinuxMount: &enabled, + }, + { + name: "SELinux mount support feature enabled, before: off, update: on", + seLinuxMountReadWriteOncePodEnabled: true, + old: driverWithSELinuxMountDisabled, + update: driverWithSELinuxMountEnabled, + wantSELinuxMount: &enabled, + }, + { + name: "SELinux mount support feature enabled, before: on, update: off", + seLinuxMountReadWriteOncePodEnabled: true, + old: driverWithSELinuxMountEnabled, + update: driverWithSELinuxMountDisabled, + wantSELinuxMount: &disabled, + }, + { + name: "SELinux mount support feature disabled, before: nil, update: on", + seLinuxMountReadWriteOncePodEnabled: false, + old: driverWithNothing, + update: driverWithSELinuxMountEnabled, + wantSELinuxMount: nil, + }, + { + name: "SELinux mount support feature disabled, before: off, update: on", + seLinuxMountReadWriteOncePodEnabled: false, + old: driverWithSELinuxMountDisabled, + update: driverWithSELinuxMountEnabled, + wantSELinuxMount: &enabled, + }, + { + name: "SELinux mount support feature enabled, before: on, update: off", + seLinuxMountReadWriteOncePodEnabled: false, + old: driverWithSELinuxMountEnabled, + update: driverWithSELinuxMountDisabled, + wantSELinuxMount: &disabled, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, test.csiInlineVolumeEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, test.seLinuxMountReadWriteOncePodEnabled)() csiDriver := test.update.DeepCopy() Strategy.PrepareForUpdate(ctx, csiDriver, test.old) @@ -276,9 +337,9 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { require.Equal(t, test.wantModes, csiDriver.Spec.VolumeLifecycleModes) require.Equal(t, test.wantTokenRequests, csiDriver.Spec.TokenRequests) require.Equal(t, test.wantRequiresRepublish, csiDriver.Spec.RequiresRepublish) + require.Equal(t, test.wantSELinuxMount, csiDriver.Spec.SELinuxMounted) }) } - } func TestCSIDriverValidation(t *testing.T) { diff --git a/staging/src/k8s.io/api/storage/v1/types.go b/staging/src/k8s.io/api/storage/v1/types.go index 4812287abf6..f57099df6dc 100644 --- a/staging/src/k8s.io/api/storage/v1/types.go +++ b/staging/src/k8s.io/api/storage/v1/types.go @@ -392,6 +392,27 @@ type CSIDriverSpec struct { // // +optional RequiresRepublish *bool `json:"requiresRepublish,omitempty" protobuf:"varint,7,opt,name=requiresRepublish"` + + // SELinuxMount specifies if the CSI driver supports "-o context" + // mount option. + // + // When "true", the CSI driver must ensure that all volumes provided by this CSI + // driver can be mounted separately with different `-o context` options. This is + // typical for storage backends that provide volumes as filesystems on block + // devices or as independent shared volumes. + // Kubernetes will call NodeStage / NodePublish with "-o context=xyz" mount + // option when mounting a ReadWriteOncePod volume used in Pod that has + // explicitly set SELinux context. In the future, it may be expanded to other + // volume AccessModes. In any case, Kubernetes will ensure that the volume is + // mounted only with a single SELinux context. + // + // When "false", Kubernetes won't pass any special SELinux mount options to the driver. + // This is typical for volumes that represent subdirectories of a bigger shared filesystem. + // + // Default is "false". + // + // +optional + SELinuxMount *bool `json:"seLinuxMount,omitempty" protobuf:"varint,8,opt,name=seLinuxMount"` } // FSGroupPolicy specifies if a CSI Driver supports modifying diff --git a/staging/src/k8s.io/api/storage/v1beta1/types.go b/staging/src/k8s.io/api/storage/v1beta1/types.go index b39414b9605..f4d09b641a9 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types.go @@ -410,6 +410,27 @@ type CSIDriverSpec struct { // // +optional RequiresRepublish *bool `json:"requiresRepublish,omitempty" protobuf:"varint,7,opt,name=requiresRepublish"` + + // SELinuxMount specifies if the CSI driver supports "-o context" + // mount option. + // + // When "true", the CSI driver must ensure that all volumes provided by this CSI + // driver can be mounted separately with different `-o context` options. This is + // typical for storage backends that provide volumes as filesystems on block + // devices or as independent shared volumes. + // Kubernetes will call NodeStage / NodePublish with "-o context=xyz" mount + // option when mounting a ReadWriteOncePod volume used in Pod that has + // explicitly set SELinux context. In the future, it may be expanded to other + // volume AccessModes. In any case, Kubernetes will ensure that the volume is + // mounted only with a single SELinux context. + // + // When "false", Kubernetes won't pass any special SELinux mount options to the driver. + // This is typical for volumes that represent subdirectories of a bigger shared filesystem. + // + // Default is "false". + // + // +optional + SELinuxMount *bool `json:"seLinuxMount,omitempty" protobuf:"varint,8,opt,name=seLinuxMount"` } // FSGroupPolicy specifies if a CSI Driver supports modifying From 189f19a6985c577cff3abbae75a94687deb9d4e0 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 6 Jun 2022 17:29:45 +0200 Subject: [PATCH 03/22] Update generation when SELinuxMount is changed --- pkg/registry/storage/csidriver/strategy.go | 12 +++++++----- pkg/registry/storage/csidriver/strategy_test.go | 8 +++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/registry/storage/csidriver/strategy.go b/pkg/registry/storage/csidriver/strategy.go index d266dfa0431..2eb87e568a3 100644 --- a/pkg/registry/storage/csidriver/strategy.go +++ b/pkg/registry/storage/csidriver/strategy.go @@ -86,15 +86,17 @@ func (csiDriverStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime. newCSIDriver.Spec.VolumeLifecycleModes = nil } - // Any changes to the mutable fields increment the generation number. - if !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.TokenRequests, newCSIDriver.Spec.TokenRequests) || !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.RequiresRepublish, newCSIDriver.Spec.RequiresRepublish) { - newCSIDriver.Generation = oldCSIDriver.Generation + 1 - } - if oldCSIDriver.Spec.SELinuxMount == nil && !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { newCSIDriver.Spec.SELinuxMount = nil } + + // Any changes to the mutable fields increment the generation number. + if !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.TokenRequests, newCSIDriver.Spec.TokenRequests) || + !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.RequiresRepublish, newCSIDriver.Spec.RequiresRepublish) || + !apiequality.Semantic.DeepEqual(oldCSIDriver.Spec.SELinuxMount, newCSIDriver.Spec.SELinuxMount) { + newCSIDriver.Generation = oldCSIDriver.Generation + 1 + } } func (csiDriverStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/storage/csidriver/strategy_test.go b/pkg/registry/storage/csidriver/strategy_test.go index e0172823188..7f7829d4ce6 100644 --- a/pkg/registry/storage/csidriver/strategy_test.go +++ b/pkg/registry/storage/csidriver/strategy_test.go @@ -287,6 +287,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithNothing, update: driverWithSELinuxMountEnabled, wantSELinuxMount: &enabled, + wantGeneration: 1, }, { name: "SELinux mount support feature enabled, before: off, update: on", @@ -294,6 +295,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithSELinuxMountDisabled, update: driverWithSELinuxMountEnabled, wantSELinuxMount: &enabled, + wantGeneration: 1, }, { name: "SELinux mount support feature enabled, before: on, update: off", @@ -301,6 +303,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithSELinuxMountEnabled, update: driverWithSELinuxMountDisabled, wantSELinuxMount: &disabled, + wantGeneration: 1, }, { name: "SELinux mount support feature disabled, before: nil, update: on", @@ -308,6 +311,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithNothing, update: driverWithSELinuxMountEnabled, wantSELinuxMount: nil, + wantGeneration: 0, }, { name: "SELinux mount support feature disabled, before: off, update: on", @@ -315,6 +319,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithSELinuxMountDisabled, update: driverWithSELinuxMountEnabled, wantSELinuxMount: &enabled, + wantGeneration: 1, }, { name: "SELinux mount support feature enabled, before: on, update: off", @@ -322,6 +327,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { old: driverWithSELinuxMountEnabled, update: driverWithSELinuxMountDisabled, wantSELinuxMount: &disabled, + wantGeneration: 1, }, } @@ -337,7 +343,7 @@ func TestCSIDriverPrepareForUpdate(t *testing.T) { require.Equal(t, test.wantModes, csiDriver.Spec.VolumeLifecycleModes) require.Equal(t, test.wantTokenRequests, csiDriver.Spec.TokenRequests) require.Equal(t, test.wantRequiresRepublish, csiDriver.Spec.RequiresRepublish) - require.Equal(t, test.wantSELinuxMount, csiDriver.Spec.SELinuxMounted) + require.Equal(t, test.wantSELinuxMount, csiDriver.Spec.SELinuxMount) }) } } From f2fd9c1c16c7ba99856d96f69c3b9bde1b811367 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 6 Jun 2022 20:08:29 +0200 Subject: [PATCH 04/22] Regenerate files --- api/openapi-spec/swagger.json | 4 + .../v3/apis__storage.k8s.io__v1_openapi.json | 4 + .../storage/v1/zz_generated.conversion.go | 2 + .../v1beta1/zz_generated.conversion.go | 2 + pkg/apis/storage/zz_generated.deepcopy.go | 5 + pkg/generated/openapi/zz_generated.openapi.go | 14 + .../src/k8s.io/api/storage/v1/generated.pb.go | 246 ++++++++++-------- .../src/k8s.io/api/storage/v1/generated.proto | 21 ++ .../storage/v1/types_swagger_doc_generated.go | 1 + .../api/storage/v1/zz_generated.deepcopy.go | 5 + .../api/storage/v1beta1/generated.pb.go | 246 ++++++++++-------- .../api/storage/v1beta1/generated.proto | 21 ++ .../v1beta1/types_swagger_doc_generated.go | 1 + .../storage/v1beta1/zz_generated.deepcopy.go | 5 + .../HEAD/storage.k8s.io.v1.CSIDriver.json | 3 +- .../HEAD/storage.k8s.io.v1.CSIDriver.pb | Bin 474 -> 476 bytes .../HEAD/storage.k8s.io.v1.CSIDriver.yaml | 1 + .../storage.k8s.io.v1beta1.CSIDriver.json | 3 +- .../HEAD/storage.k8s.io.v1beta1.CSIDriver.pb | Bin 479 -> 481 bytes .../storage.k8s.io.v1beta1.CSIDriver.yaml | 1 + .../applyconfigurations/internal/internal.go | 6 + .../storage/v1/csidriverspec.go | 9 + .../storage/v1beta1/csidriverspec.go | 9 + 23 files changed, 397 insertions(+), 212 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index c64b20773db..b1ccfd6aa59 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -12832,6 +12832,10 @@ "description": "RequiresRepublish indicates the CSI driver wants `NodePublishVolume` being periodically called to reflect any possible change in the mounted volume. This field defaults to false.\n\nNote: After a successful initial NodePublishVolume call, subsequent calls to NodePublishVolume should only update the contents of the volume. New mount points will not be seen by a running container.", "type": "boolean" }, + "seLinuxMount": { + "description": "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", + "type": "boolean" + }, "storageCapacity": { "description": "If set to true, storageCapacity indicates that the CSI volume driver wants pod scheduling to consider the storage capacity that the driver deployment will report by creating CSIStorageCapacity objects with capacity information.\n\nThe check can be enabled immediately when deploying a driver. In that case, provisioning new volumes with late binding will pause until the driver deployment has published some suitable CSIStorageCapacity object.\n\nAlternatively, the driver can be deployed with the field unset or false and it can be flipped later when storage capacity information has been published.\n\nThis field was immutable in Kubernetes <= 1.22 and now is mutable.", "type": "boolean" diff --git a/api/openapi-spec/v3/apis__storage.k8s.io__v1_openapi.json b/api/openapi-spec/v3/apis__storage.k8s.io__v1_openapi.json index 9bc9ae81224..f767836eac5 100644 --- a/api/openapi-spec/v3/apis__storage.k8s.io__v1_openapi.json +++ b/api/openapi-spec/v3/apis__storage.k8s.io__v1_openapi.json @@ -1280,6 +1280,10 @@ "description": "RequiresRepublish indicates the CSI driver wants `NodePublishVolume` being periodically called to reflect any possible change in the mounted volume. This field defaults to false.\n\nNote: After a successful initial NodePublishVolume call, subsequent calls to NodePublishVolume should only update the contents of the volume. New mount points will not be seen by a running container.", "type": "boolean" }, + "seLinuxMount": { + "description": "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", + "type": "boolean" + }, "storageCapacity": { "description": "If set to true, storageCapacity indicates that the CSI volume driver wants pod scheduling to consider the storage capacity that the driver deployment will report by creating CSIStorageCapacity objects with capacity information.\n\nThe check can be enabled immediately when deploying a driver. In that case, provisioning new volumes with late binding will pause until the driver deployment has published some suitable CSIStorageCapacity object.\n\nAlternatively, the driver can be deployed with the field unset or false and it can be flipped later when storage capacity information has been published.\n\nThis field was immutable in Kubernetes <= 1.22 and now is mutable.", "type": "boolean" diff --git a/pkg/apis/storage/v1/zz_generated.conversion.go b/pkg/apis/storage/v1/zz_generated.conversion.go index 3c7e174d1d9..8fefe5fefd3 100644 --- a/pkg/apis/storage/v1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1/zz_generated.conversion.go @@ -311,6 +311,7 @@ func autoConvert_v1_CSIDriverSpec_To_storage_CSIDriverSpec(in *v1.CSIDriverSpec, out.FSGroupPolicy = (*storage.FSGroupPolicy)(unsafe.Pointer(in.FSGroupPolicy)) out.TokenRequests = *(*[]storage.TokenRequest)(unsafe.Pointer(&in.TokenRequests)) out.RequiresRepublish = (*bool)(unsafe.Pointer(in.RequiresRepublish)) + out.SELinuxMount = (*bool)(unsafe.Pointer(in.SELinuxMount)) return nil } @@ -327,6 +328,7 @@ func autoConvert_storage_CSIDriverSpec_To_v1_CSIDriverSpec(in *storage.CSIDriver out.StorageCapacity = (*bool)(unsafe.Pointer(in.StorageCapacity)) out.TokenRequests = *(*[]v1.TokenRequest)(unsafe.Pointer(&in.TokenRequests)) out.RequiresRepublish = (*bool)(unsafe.Pointer(in.RequiresRepublish)) + out.SELinuxMount = (*bool)(unsafe.Pointer(in.SELinuxMount)) return nil } diff --git a/pkg/apis/storage/v1beta1/zz_generated.conversion.go b/pkg/apis/storage/v1beta1/zz_generated.conversion.go index be7360d829c..6b3580aa0df 100644 --- a/pkg/apis/storage/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1beta1/zz_generated.conversion.go @@ -311,6 +311,7 @@ func autoConvert_v1beta1_CSIDriverSpec_To_storage_CSIDriverSpec(in *v1beta1.CSID out.FSGroupPolicy = (*storage.FSGroupPolicy)(unsafe.Pointer(in.FSGroupPolicy)) out.TokenRequests = *(*[]storage.TokenRequest)(unsafe.Pointer(&in.TokenRequests)) out.RequiresRepublish = (*bool)(unsafe.Pointer(in.RequiresRepublish)) + out.SELinuxMount = (*bool)(unsafe.Pointer(in.SELinuxMount)) return nil } @@ -327,6 +328,7 @@ func autoConvert_storage_CSIDriverSpec_To_v1beta1_CSIDriverSpec(in *storage.CSID out.StorageCapacity = (*bool)(unsafe.Pointer(in.StorageCapacity)) out.TokenRequests = *(*[]v1beta1.TokenRequest)(unsafe.Pointer(&in.TokenRequests)) out.RequiresRepublish = (*bool)(unsafe.Pointer(in.RequiresRepublish)) + out.SELinuxMount = (*bool)(unsafe.Pointer(in.SELinuxMount)) return nil } diff --git a/pkg/apis/storage/zz_generated.deepcopy.go b/pkg/apis/storage/zz_generated.deepcopy.go index e480130b196..661df7a349e 100644 --- a/pkg/apis/storage/zz_generated.deepcopy.go +++ b/pkg/apis/storage/zz_generated.deepcopy.go @@ -127,6 +127,11 @@ func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = new(bool) **out = **in } + if in.SELinuxMount != nil { + in, out := &in.SELinuxMount, &out.SELinuxMount + *out = new(bool) + **out = **in + } return } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index e4f8588472d..c856672ebb5 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -38969,6 +38969,13 @@ func schema_k8sio_api_storage_v1_CSIDriverSpec(ref common.ReferenceCallback) com Format: "", }, }, + "seLinuxMount": { + SchemaProps: spec.SchemaProps{ + Description: "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", + Type: []string{"boolean"}, + Format: "", + }, + }, }, }, }, @@ -40292,6 +40299,13 @@ func schema_k8sio_api_storage_v1beta1_CSIDriverSpec(ref common.ReferenceCallback Format: "", }, }, + "seLinuxMount": { + SchemaProps: spec.SchemaProps{ + Description: "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", + Type: []string{"boolean"}, + Format: "", + }, + }, }, }, }, diff --git a/staging/src/k8s.io/api/storage/v1/generated.pb.go b/staging/src/k8s.io/api/storage/v1/generated.pb.go index a648c426aab..d36497432de 100644 --- a/staging/src/k8s.io/api/storage/v1/generated.pb.go +++ b/staging/src/k8s.io/api/storage/v1/generated.pb.go @@ -609,111 +609,112 @@ func init() { } var fileDescriptor_3b530c1983504d8d = []byte{ - // 1651 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xbd, 0x6f, 0x1b, 0xcb, - 0x11, 0xd7, 0x89, 0xd4, 0xd7, 0x52, 0xb2, 0xa4, 0x95, 0xe4, 0x30, 0x2a, 0x48, 0xe1, 0x6c, 0x24, - 0xb2, 0x13, 0x1f, 0x6d, 0xd9, 0x31, 0x0c, 0x07, 0x0e, 0xa0, 0x93, 0xe8, 0x58, 0x88, 0x28, 0x29, - 0x4b, 0xc5, 0x30, 0x82, 0x24, 0xf0, 0xea, 0x6e, 0x45, 0xad, 0xc5, 0xfb, 0xf0, 0xed, 0x92, 0x31, - 0x53, 0x25, 0x4d, 0xba, 0x00, 0x49, 0x1b, 0xe4, 0x8f, 0x48, 0x80, 0xa4, 0x49, 0x99, 0x22, 0x70, - 0x3a, 0x23, 0x95, 0x2b, 0xe2, 0x99, 0xaf, 0x7e, 0xaf, 0x7c, 0x85, 0xaa, 0x87, 0xdd, 0x5b, 0xf2, - 0x3e, 0x78, 0x94, 0xa5, 0x86, 0x1d, 0x77, 0x67, 0xe6, 0x37, 0xb3, 0x3b, 0xbf, 0x99, 0x9d, 0x23, - 0xf8, 0xc9, 0xf9, 0x13, 0x66, 0x50, 0xaf, 0x72, 0xde, 0x3a, 0x21, 0x81, 0x4b, 0x38, 0x61, 0x95, - 0x36, 0x71, 0x6d, 0x2f, 0xa8, 0x28, 0x01, 0xf6, 0x69, 0x85, 0x71, 0x2f, 0xc0, 0x0d, 0x52, 0x69, - 0x3f, 0xa8, 0x34, 0x88, 0x4b, 0x02, 0xcc, 0x89, 0x6d, 0xf8, 0x81, 0xc7, 0x3d, 0xb8, 0x16, 0xaa, - 0x19, 0xd8, 0xa7, 0x86, 0x52, 0x33, 0xda, 0x0f, 0xd6, 0xef, 0x35, 0x28, 0x3f, 0x6b, 0x9d, 0x18, - 0x96, 0xe7, 0x54, 0x1a, 0x5e, 0xc3, 0xab, 0x48, 0xed, 0x93, 0xd6, 0xa9, 0x5c, 0xc9, 0x85, 0xfc, - 0x15, 0xa2, 0xac, 0xeb, 0x31, 0x67, 0x96, 0x17, 0x64, 0x79, 0x5a, 0x7f, 0x14, 0xe9, 0x38, 0xd8, - 0x3a, 0xa3, 0x2e, 0x09, 0x3a, 0x15, 0xff, 0xbc, 0x21, 0x8d, 0x02, 0xc2, 0xbc, 0x56, 0x60, 0x91, - 0x6b, 0x59, 0xb1, 0x8a, 0x43, 0x38, 0xce, 0xf2, 0x55, 0x19, 0x65, 0x15, 0xb4, 0x5c, 0x4e, 0x9d, - 0x61, 0x37, 0x8f, 0x3f, 0x67, 0xc0, 0xac, 0x33, 0xe2, 0xe0, 0xb4, 0x9d, 0xfe, 0x2f, 0x0d, 0xcc, - 0xed, 0xd4, 0xf7, 0x76, 0x03, 0xda, 0x26, 0x01, 0x7c, 0x0d, 0x66, 0x45, 0x44, 0x36, 0xe6, 0xb8, - 0xa8, 0x6d, 0x68, 0x9b, 0x85, 0xad, 0xfb, 0x46, 0x74, 0xbf, 0x03, 0x60, 0xc3, 0x3f, 0x6f, 0x88, - 0x0d, 0x66, 0x08, 0x6d, 0xa3, 0xfd, 0xc0, 0x38, 0x3c, 0x79, 0x43, 0x2c, 0x5e, 0x23, 0x1c, 0x9b, - 0xf0, 0x7d, 0xb7, 0x3c, 0xd1, 0xeb, 0x96, 0x41, 0xb4, 0x87, 0x06, 0xa8, 0xf0, 0x39, 0xc8, 0x33, - 0x9f, 0x58, 0xc5, 0x49, 0x89, 0x7e, 0xdb, 0xc8, 0xcc, 0x9e, 0x31, 0x88, 0xa8, 0xee, 0x13, 0xcb, - 0x9c, 0x57, 0x88, 0x79, 0xb1, 0x42, 0xd2, 0x5e, 0xff, 0xa7, 0x06, 0x16, 0x06, 0x5a, 0xfb, 0x94, - 0x71, 0xf8, 0xab, 0xa1, 0xd8, 0x8d, 0xab, 0xc5, 0x2e, 0xac, 0x65, 0xe4, 0x4b, 0xca, 0xcf, 0x6c, - 0x7f, 0x27, 0x16, 0x77, 0x15, 0x4c, 0x51, 0x4e, 0x1c, 0x56, 0x9c, 0xdc, 0xc8, 0x6d, 0x16, 0xb6, - 0x36, 0x3e, 0x17, 0xb8, 0xb9, 0xa0, 0xc0, 0xa6, 0xf6, 0x84, 0x19, 0x0a, 0xad, 0xf5, 0xbf, 0xe5, - 0x63, 0x61, 0x8b, 0xe3, 0xc0, 0xa7, 0xe0, 0x06, 0xe6, 0x1c, 0x5b, 0x67, 0x88, 0xbc, 0x6d, 0xd1, - 0x80, 0xd8, 0x32, 0xf8, 0x59, 0x13, 0xf6, 0xba, 0xe5, 0x1b, 0xdb, 0x09, 0x09, 0x4a, 0x69, 0x0a, - 0x5b, 0xdf, 0xb3, 0xf7, 0xdc, 0x53, 0xef, 0xd0, 0xad, 0x79, 0x2d, 0x97, 0xcb, 0x6b, 0x55, 0xb6, - 0x47, 0x09, 0x09, 0x4a, 0x69, 0x42, 0x0b, 0xac, 0xb6, 0xbd, 0x66, 0xcb, 0x21, 0xfb, 0xf4, 0x94, - 0x58, 0x1d, 0xab, 0x49, 0x6a, 0x9e, 0x4d, 0x58, 0x31, 0xb7, 0x91, 0xdb, 0x9c, 0x33, 0x2b, 0xbd, - 0x6e, 0x79, 0xf5, 0x65, 0x86, 0xfc, 0xa2, 0x5b, 0x5e, 0xc9, 0xd8, 0x47, 0x99, 0x60, 0xf0, 0x19, - 0x58, 0x54, 0x97, 0xb3, 0x83, 0x7d, 0x6c, 0x51, 0xde, 0x29, 0xe6, 0x65, 0x84, 0x2b, 0xbd, 0x6e, - 0x79, 0xb1, 0x9e, 0x14, 0xa1, 0xb4, 0x2e, 0x7c, 0x01, 0x16, 0x4e, 0xd9, 0x4f, 0x03, 0xaf, 0xe5, - 0x1f, 0x79, 0x4d, 0x6a, 0x75, 0x8a, 0x53, 0x1b, 0xda, 0xe6, 0x9c, 0xa9, 0xf7, 0xba, 0xe5, 0x85, - 0xe7, 0xf5, 0x98, 0xe0, 0x22, 0xbd, 0x81, 0x92, 0x86, 0xf0, 0x35, 0x58, 0xe0, 0xde, 0x39, 0x71, - 0xc5, 0xd5, 0x11, 0xc6, 0x59, 0x71, 0x5a, 0xa6, 0xf1, 0xd6, 0x88, 0x34, 0x1e, 0xc7, 0x74, 0xcd, - 0x35, 0x95, 0xc9, 0x85, 0xf8, 0x2e, 0x43, 0x49, 0x40, 0xb8, 0x03, 0x96, 0x83, 0x30, 0x2f, 0x0c, - 0x11, 0xbf, 0x75, 0xd2, 0xa4, 0xec, 0xac, 0x38, 0x23, 0x0f, 0xbb, 0xd6, 0xeb, 0x96, 0x97, 0x51, - 0x5a, 0x88, 0x86, 0xf5, 0xf5, 0x7f, 0x68, 0x60, 0x66, 0xa7, 0xbe, 0x77, 0xe0, 0xd9, 0x64, 0x0c, - 0xb5, 0xb8, 0x9b, 0xa8, 0x45, 0x7d, 0x34, 0xa5, 0x45, 0x3c, 0x23, 0x2b, 0xf1, 0xeb, 0xb0, 0x12, - 0x85, 0x8e, 0xea, 0x22, 0x1b, 0x20, 0xef, 0x62, 0x87, 0xc8, 0xa8, 0xe7, 0x22, 0x9b, 0x03, 0xec, - 0x10, 0x24, 0x25, 0xf0, 0x7b, 0x60, 0xda, 0xf5, 0x6c, 0xb2, 0xb7, 0x2b, 0x7d, 0xcf, 0x99, 0x37, - 0x94, 0xce, 0xf4, 0x81, 0xdc, 0x45, 0x4a, 0x0a, 0x1f, 0x81, 0x79, 0xee, 0xf9, 0x5e, 0xd3, 0x6b, - 0x74, 0x7e, 0x46, 0x3a, 0x7d, 0x72, 0x2e, 0xf5, 0xba, 0xe5, 0xf9, 0xe3, 0xd8, 0x3e, 0x4a, 0x68, - 0xc1, 0x5f, 0x83, 0x02, 0x6e, 0x36, 0x3d, 0x0b, 0x73, 0x7c, 0xd2, 0x24, 0x92, 0x71, 0x85, 0xad, - 0xbb, 0x23, 0x8e, 0x17, 0x92, 0x59, 0xf8, 0x45, 0xaa, 0x85, 0x33, 0x73, 0xb1, 0xd7, 0x2d, 0x17, - 0xb6, 0x23, 0x08, 0x14, 0xc7, 0xd3, 0xff, 0xae, 0x81, 0x82, 0x3a, 0xf0, 0x18, 0x1a, 0xcf, 0x4e, - 0xb2, 0xf1, 0x94, 0x2e, 0xcf, 0xd2, 0x88, 0xb6, 0xf3, 0x9b, 0x41, 0xc4, 0xb2, 0xe7, 0x1c, 0x82, - 0x19, 0x5b, 0xa6, 0x8a, 0x15, 0x35, 0x89, 0x7a, 0xfb, 0x72, 0x54, 0xd5, 0xd2, 0x16, 0x15, 0xf6, - 0x4c, 0xb8, 0x66, 0xa8, 0x8f, 0xa2, 0x7f, 0x93, 0x03, 0x70, 0xa7, 0xbe, 0x97, 0x2a, 0xe8, 0x31, - 0x50, 0x98, 0x82, 0x79, 0x41, 0x95, 0x3e, 0x19, 0x14, 0x95, 0x1f, 0x5e, 0xf1, 0xfe, 0xf1, 0x09, - 0x69, 0xd6, 0x49, 0x93, 0x58, 0xdc, 0x0b, 0x42, 0x56, 0x1d, 0xc4, 0xc0, 0x50, 0x02, 0x1a, 0xee, - 0x82, 0xa5, 0x7e, 0x7f, 0x6a, 0x62, 0xc6, 0x04, 0x9b, 0x8b, 0x39, 0xc9, 0xde, 0xa2, 0x0a, 0x71, - 0xa9, 0x9e, 0x92, 0xa3, 0x21, 0x0b, 0xf8, 0x0a, 0xcc, 0x5a, 0xf1, 0x56, 0xf8, 0x19, 0xb2, 0x18, - 0xfd, 0xb9, 0xc2, 0xf8, 0x79, 0x0b, 0xbb, 0x9c, 0xf2, 0x8e, 0x39, 0x2f, 0x88, 0x32, 0xe8, 0x99, - 0x03, 0x34, 0xc8, 0xc0, 0xb2, 0x83, 0xdf, 0x51, 0xa7, 0xe5, 0x84, 0x94, 0xae, 0xd3, 0xdf, 0x11, - 0xd9, 0x30, 0xaf, 0xef, 0x42, 0x36, 0xac, 0x5a, 0x1a, 0x0c, 0x0d, 0xe3, 0xeb, 0xff, 0xd5, 0xc0, - 0xcd, 0xe1, 0xc4, 0x8f, 0xa1, 0x2c, 0x0e, 0x92, 0x65, 0x71, 0x67, 0x34, 0x81, 0x53, 0xb1, 0x8d, - 0xa8, 0x90, 0x3f, 0x4d, 0x83, 0xf9, 0x78, 0xfa, 0xc6, 0xc0, 0xdd, 0x1f, 0x81, 0x82, 0x1f, 0x78, - 0x6d, 0xca, 0xa8, 0xe7, 0x92, 0x40, 0x75, 0xc2, 0x15, 0x65, 0x52, 0x38, 0x8a, 0x44, 0x28, 0xae, - 0x07, 0x1b, 0x00, 0xf8, 0x38, 0xc0, 0x0e, 0xe1, 0xa2, 0x7e, 0x73, 0xf2, 0xf8, 0x0f, 0x47, 0x1c, - 0x3f, 0x7e, 0x22, 0xe3, 0x68, 0x60, 0x55, 0x75, 0x79, 0xd0, 0x89, 0xa2, 0x8b, 0x04, 0x28, 0x06, - 0x0d, 0xcf, 0xc1, 0x42, 0x40, 0xac, 0x26, 0xa6, 0x8e, 0x7a, 0x7d, 0xf3, 0x32, 0xc2, 0xaa, 0x78, - 0x0a, 0x51, 0x5c, 0x70, 0xd1, 0x2d, 0xdf, 0x1f, 0x9e, 0x9f, 0x8d, 0x23, 0x12, 0x30, 0xca, 0x38, - 0x71, 0x79, 0x48, 0x98, 0x84, 0x0d, 0x4a, 0x62, 0x8b, 0x4e, 0xef, 0x88, 0xb9, 0xe4, 0xd0, 0xe7, - 0xd4, 0x73, 0x59, 0x71, 0x2a, 0xea, 0xf4, 0xb5, 0xd8, 0x3e, 0x4a, 0x68, 0xc1, 0x7d, 0xb0, 0x2a, - 0x3a, 0xf3, 0x6f, 0x43, 0x07, 0xd5, 0x77, 0x3e, 0x76, 0xc5, 0x2d, 0x15, 0xa7, 0xe5, 0xbb, 0x5b, - 0x14, 0x43, 0xcc, 0x76, 0x86, 0x1c, 0x65, 0x5a, 0xc1, 0x57, 0x60, 0x39, 0x9c, 0x62, 0x4c, 0xea, - 0xda, 0xd4, 0x6d, 0x88, 0x19, 0x46, 0x3e, 0xe1, 0x73, 0xe6, 0x5d, 0x51, 0x11, 0x2f, 0xd3, 0xc2, - 0x8b, 0xac, 0x4d, 0x34, 0x0c, 0x02, 0xdf, 0x82, 0x65, 0xe9, 0x91, 0xd8, 0xaa, 0x9d, 0x50, 0xc2, - 0x8a, 0xb3, 0x32, 0x75, 0x9b, 0xf1, 0xd4, 0x89, 0xab, 0x0b, 0xe7, 0x8f, 0xb0, 0xe9, 0xf4, 0x9b, - 0xd3, 0x31, 0x09, 0x1c, 0xf3, 0xbb, 0x2a, 0x5f, 0xcb, 0xdb, 0x69, 0x28, 0x34, 0x8c, 0xbe, 0xfe, - 0x0c, 0x2c, 0xa6, 0x12, 0x0e, 0x97, 0x40, 0xee, 0x9c, 0x74, 0xc2, 0x67, 0x19, 0x89, 0x9f, 0x70, - 0x15, 0x4c, 0xb5, 0x71, 0xb3, 0x45, 0x42, 0xf2, 0xa1, 0x70, 0xf1, 0x74, 0xf2, 0x89, 0xa6, 0xff, - 0x5b, 0x03, 0x89, 0x76, 0x36, 0x86, 0x92, 0x7e, 0x91, 0x2c, 0xe9, 0x5b, 0x57, 0xe0, 0xf4, 0x88, - 0x62, 0xfe, 0x83, 0x06, 0xe6, 0xe3, 0xc3, 0x1a, 0xfc, 0x21, 0x98, 0xc5, 0x2d, 0x9b, 0x12, 0xd7, - 0xea, 0x4f, 0x25, 0x83, 0x40, 0xb6, 0xd5, 0x3e, 0x1a, 0x68, 0x88, 0x51, 0x8e, 0xbc, 0xf3, 0x69, - 0x80, 0x05, 0xc9, 0xea, 0xc4, 0xf2, 0x5c, 0x9b, 0xc9, 0x1b, 0xca, 0x85, 0x9d, 0xb1, 0x9a, 0x16, - 0xa2, 0x61, 0x7d, 0xfd, 0xaf, 0x93, 0x60, 0x29, 0xe4, 0x46, 0x38, 0xc4, 0x3b, 0xc4, 0xe5, 0x63, - 0x68, 0x2a, 0xb5, 0xc4, 0x4c, 0xf7, 0x83, 0x4b, 0x87, 0x9e, 0x28, 0xb0, 0x51, 0xc3, 0x1d, 0xfc, - 0x05, 0x98, 0x66, 0x1c, 0xf3, 0x16, 0x93, 0x4f, 0x5d, 0x61, 0xeb, 0xde, 0x55, 0x01, 0xa5, 0x51, - 0x34, 0xd7, 0x85, 0x6b, 0xa4, 0xc0, 0xf4, 0xff, 0x68, 0x60, 0x35, 0x6d, 0x32, 0x06, 0x86, 0xed, - 0x27, 0x19, 0xf6, 0xfd, 0x2b, 0x1e, 0x66, 0x04, 0xcb, 0xfe, 0xaf, 0x81, 0x9b, 0x43, 0xe7, 0x96, - 0x2f, 0xa9, 0xe8, 0x4b, 0x7e, 0xaa, 0xfb, 0x1d, 0x44, 0x13, 0xb1, 0xec, 0x4b, 0x47, 0x19, 0x72, - 0x94, 0x69, 0x05, 0xdf, 0x80, 0x25, 0xea, 0x36, 0xa9, 0x4b, 0xd4, 0xc3, 0x1b, 0xe5, 0x37, 0xb3, - 0x79, 0xa4, 0x91, 0x65, 0x72, 0x57, 0xc5, 0x7c, 0xb2, 0x97, 0x42, 0x41, 0x43, 0xb8, 0xfa, 0xff, - 0x32, 0x32, 0x23, 0x67, 0x46, 0x51, 0x42, 0x72, 0x87, 0x04, 0x43, 0x25, 0xa4, 0xf6, 0xd1, 0x40, - 0x43, 0xf2, 0x46, 0x5e, 0x85, 0x0a, 0xf4, 0xca, 0xbc, 0x91, 0x46, 0x31, 0xde, 0xc8, 0x35, 0x52, - 0x60, 0x22, 0x08, 0x31, 0x93, 0xc5, 0x66, 0xaf, 0x41, 0x10, 0x07, 0x6a, 0x1f, 0x0d, 0x34, 0xf4, - 0xaf, 0x72, 0x19, 0x09, 0x92, 0x04, 0x8c, 0x9d, 0xa6, 0xff, 0xbd, 0x9d, 0x3e, 0x8d, 0x3d, 0x38, - 0x8d, 0x0d, 0xff, 0xa2, 0x01, 0x88, 0x07, 0x10, 0xb5, 0x3e, 0x41, 0x43, 0x16, 0x55, 0xaf, 0x55, - 0x12, 0xc6, 0xf6, 0x10, 0x4e, 0xf8, 0x1a, 0xaf, 0x2b, 0xff, 0x70, 0x58, 0x01, 0x65, 0x38, 0x87, - 0x36, 0x28, 0x84, 0xbb, 0xd5, 0x20, 0xf0, 0x02, 0x55, 0x9e, 0xfa, 0xa5, 0xb1, 0x48, 0x4d, 0xb3, - 0x24, 0x3f, 0x6e, 0x22, 0xd3, 0x8b, 0x6e, 0xb9, 0x10, 0x93, 0xa3, 0x38, 0xac, 0xf0, 0x62, 0x93, - 0xc8, 0x4b, 0xfe, 0x7a, 0x5e, 0x76, 0xc9, 0x68, 0x2f, 0x31, 0xd8, 0xf5, 0x2a, 0xf8, 0xce, 0x88, - 0x6b, 0xb9, 0xd6, 0x9b, 0xf5, 0x47, 0x0d, 0xc4, 0x7d, 0xc0, 0x7d, 0x90, 0xe7, 0x54, 0x55, 0x5d, - 0xf2, 0x03, 0xf0, 0x92, 0x46, 0x72, 0x4c, 0x1d, 0x12, 0xb5, 0x42, 0xb1, 0x42, 0x12, 0x05, 0xde, - 0x01, 0x33, 0x0e, 0x61, 0x0c, 0x37, 0x94, 0xe7, 0xe8, 0x73, 0xa8, 0x16, 0x6e, 0xa3, 0xbe, 0x5c, - 0x7f, 0x0c, 0x56, 0x32, 0x3e, 0x2b, 0x61, 0x19, 0x4c, 0x59, 0xf2, 0x5f, 0x1a, 0x11, 0xd0, 0x94, - 0x39, 0x27, 0x3a, 0xca, 0x8e, 0xfc, 0x73, 0x26, 0xdc, 0x37, 0x7f, 0xfc, 0xfe, 0x53, 0x69, 0xe2, - 0xc3, 0xa7, 0xd2, 0xc4, 0xc7, 0x4f, 0xa5, 0x89, 0xdf, 0xf7, 0x4a, 0xda, 0xfb, 0x5e, 0x49, 0xfb, - 0xd0, 0x2b, 0x69, 0x1f, 0x7b, 0x25, 0xed, 0x8b, 0x5e, 0x49, 0xfb, 0xf3, 0x97, 0xa5, 0x89, 0x5f, - 0xae, 0x65, 0xfe, 0x31, 0xfa, 0x6d, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd7, 0x6e, 0x72, 0x7b, 0x49, - 0x15, 0x00, 0x00, + // 1670 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xcd, 0x73, 0x1b, 0x4b, + 0x11, 0xf7, 0x5a, 0xf2, 0xd7, 0xc8, 0x8e, 0xed, 0xb1, 0xfd, 0x10, 0x3e, 0x48, 0xae, 0x7d, 0xaf, + 0xc0, 0xef, 0xc1, 0x5b, 0xbd, 0x38, 0x21, 0x95, 0x0a, 0x15, 0xaa, 0xbc, 0xb6, 0x42, 0x5c, 0x58, + 0xb6, 0x19, 0x99, 0x54, 0x8a, 0x02, 0x2a, 0xe3, 0xdd, 0xb1, 0x3c, 0xb1, 0xf6, 0x23, 0x3b, 0xb3, + 0xc2, 0xe2, 0x04, 0x17, 0x6e, 0x54, 0xc1, 0x95, 0xbf, 0x02, 0xaa, 0xe0, 0xc2, 0x91, 0x03, 0x15, + 0x6e, 0x29, 0x4e, 0x39, 0xa9, 0x88, 0x38, 0xc3, 0x91, 0x83, 0x4f, 0xaf, 0x66, 0x76, 0xa4, 0xfd, + 0xd0, 0xca, 0xb1, 0x2f, 0xba, 0x69, 0xa6, 0xbb, 0x7f, 0xdd, 0x33, 0xdd, 0xfd, 0x9b, 0x5e, 0x81, + 0x1f, 0x5c, 0x3e, 0x66, 0x06, 0xf5, 0x6a, 0x97, 0xe1, 0x19, 0x09, 0x5c, 0xc2, 0x09, 0xab, 0x75, + 0x88, 0x6b, 0x7b, 0x41, 0x4d, 0x09, 0xb0, 0x4f, 0x6b, 0x8c, 0x7b, 0x01, 0x6e, 0x91, 0x5a, 0xe7, + 0x7e, 0xad, 0x45, 0x5c, 0x12, 0x60, 0x4e, 0x6c, 0xc3, 0x0f, 0x3c, 0xee, 0xc1, 0x8d, 0x48, 0xcd, + 0xc0, 0x3e, 0x35, 0x94, 0x9a, 0xd1, 0xb9, 0xbf, 0xf9, 0x65, 0x8b, 0xf2, 0x8b, 0xf0, 0xcc, 0xb0, + 0x3c, 0xa7, 0xd6, 0xf2, 0x5a, 0x5e, 0x4d, 0x6a, 0x9f, 0x85, 0xe7, 0x72, 0x25, 0x17, 0xf2, 0x57, + 0x84, 0xb2, 0xa9, 0x27, 0x9c, 0x59, 0x5e, 0x90, 0xe7, 0x69, 0xf3, 0x61, 0xac, 0xe3, 0x60, 0xeb, + 0x82, 0xba, 0x24, 0xe8, 0xd6, 0xfc, 0xcb, 0x96, 0x34, 0x0a, 0x08, 0xf3, 0xc2, 0xc0, 0x22, 0x77, + 0xb2, 0x62, 0x35, 0x87, 0x70, 0x9c, 0xe7, 0xab, 0x36, 0xce, 0x2a, 0x08, 0x5d, 0x4e, 0x9d, 0x51, + 0x37, 0x8f, 0x3e, 0x66, 0xc0, 0xac, 0x0b, 0xe2, 0xe0, 0xac, 0x9d, 0xfe, 0x57, 0x0d, 0x2c, 0xec, + 0x35, 0x0f, 0xf6, 0x03, 0xda, 0x21, 0x01, 0x7c, 0x05, 0xe6, 0x45, 0x44, 0x36, 0xe6, 0xb8, 0xac, + 0x6d, 0x69, 0xdb, 0xa5, 0x9d, 0xaf, 0x8c, 0xf8, 0x7e, 0x87, 0xc0, 0x86, 0x7f, 0xd9, 0x12, 0x1b, + 0xcc, 0x10, 0xda, 0x46, 0xe7, 0xbe, 0x71, 0x7c, 0xf6, 0x9a, 0x58, 0xbc, 0x41, 0x38, 0x36, 0xe1, + 0xdb, 0x5e, 0x75, 0xaa, 0xdf, 0xab, 0x82, 0x78, 0x0f, 0x0d, 0x51, 0xe1, 0x33, 0x50, 0x64, 0x3e, + 0xb1, 0xca, 0xd3, 0x12, 0xfd, 0x33, 0x23, 0x37, 0x7b, 0xc6, 0x30, 0xa2, 0xa6, 0x4f, 0x2c, 0x73, + 0x51, 0x21, 0x16, 0xc5, 0x0a, 0x49, 0x7b, 0xfd, 0x2f, 0x1a, 0x58, 0x1a, 0x6a, 0x1d, 0x52, 0xc6, + 0xe1, 0xcf, 0x46, 0x62, 0x37, 0x6e, 0x17, 0xbb, 0xb0, 0x96, 0x91, 0xaf, 0x28, 0x3f, 0xf3, 0x83, + 0x9d, 0x44, 0xdc, 0x75, 0x30, 0x43, 0x39, 0x71, 0x58, 0x79, 0x7a, 0xab, 0xb0, 0x5d, 0xda, 0xd9, + 0xfa, 0x58, 0xe0, 0xe6, 0x92, 0x02, 0x9b, 0x39, 0x10, 0x66, 0x28, 0xb2, 0xd6, 0xff, 0x55, 0x4c, + 0x84, 0x2d, 0x8e, 0x03, 0x9f, 0x80, 0x7b, 0x98, 0x73, 0x6c, 0x5d, 0x20, 0xf2, 0x26, 0xa4, 0x01, + 0xb1, 0x65, 0xf0, 0xf3, 0x26, 0xec, 0xf7, 0xaa, 0xf7, 0x76, 0x53, 0x12, 0x94, 0xd1, 0x14, 0xb6, + 0xbe, 0x67, 0x1f, 0xb8, 0xe7, 0xde, 0xb1, 0xdb, 0xf0, 0x42, 0x97, 0xcb, 0x6b, 0x55, 0xb6, 0x27, + 0x29, 0x09, 0xca, 0x68, 0x42, 0x0b, 0xac, 0x77, 0xbc, 0x76, 0xe8, 0x90, 0x43, 0x7a, 0x4e, 0xac, + 0xae, 0xd5, 0x26, 0x0d, 0xcf, 0x26, 0xac, 0x5c, 0xd8, 0x2a, 0x6c, 0x2f, 0x98, 0xb5, 0x7e, 0xaf, + 0xba, 0xfe, 0x22, 0x47, 0x7e, 0xdd, 0xab, 0xae, 0xe5, 0xec, 0xa3, 0x5c, 0x30, 0xf8, 0x14, 0x2c, + 0xab, 0xcb, 0xd9, 0xc3, 0x3e, 0xb6, 0x28, 0xef, 0x96, 0x8b, 0x32, 0xc2, 0xb5, 0x7e, 0xaf, 0xba, + 0xdc, 0x4c, 0x8b, 0x50, 0x56, 0x17, 0x3e, 0x07, 0x4b, 0xe7, 0xec, 0x87, 0x81, 0x17, 0xfa, 0x27, + 0x5e, 0x9b, 0x5a, 0xdd, 0xf2, 0xcc, 0x96, 0xb6, 0xbd, 0x60, 0xea, 0xfd, 0x5e, 0x75, 0xe9, 0x59, + 0x33, 0x21, 0xb8, 0xce, 0x6e, 0xa0, 0xb4, 0x21, 0x7c, 0x05, 0x96, 0xb8, 0x77, 0x49, 0x5c, 0x71, + 0x75, 0x84, 0x71, 0x56, 0x9e, 0x95, 0x69, 0xfc, 0x74, 0x4c, 0x1a, 0x4f, 0x13, 0xba, 0xe6, 0x86, + 0xca, 0xe4, 0x52, 0x72, 0x97, 0xa1, 0x34, 0x20, 0xdc, 0x03, 0xab, 0x41, 0x94, 0x17, 0x86, 0x88, + 0x1f, 0x9e, 0xb5, 0x29, 0xbb, 0x28, 0xcf, 0xc9, 0xc3, 0x6e, 0xf4, 0x7b, 0xd5, 0x55, 0x94, 0x15, + 0xa2, 0x51, 0x7d, 0xf8, 0x10, 0x2c, 0x32, 0x72, 0x48, 0xdd, 0xf0, 0x2a, 0x4a, 0xe7, 0xbc, 0xb4, + 0x5f, 0xe9, 0xf7, 0xaa, 0x8b, 0xcd, 0x7a, 0xbc, 0x8f, 0x52, 0x5a, 0xfa, 0x9f, 0x35, 0x30, 0xb7, + 0xd7, 0x3c, 0x38, 0xf2, 0x6c, 0x32, 0x81, 0x0e, 0xde, 0x4f, 0x75, 0xb0, 0x3e, 0xbe, 0x11, 0x44, + 0x3c, 0x63, 0xfb, 0xf7, 0x7f, 0x51, 0xff, 0x0a, 0x1d, 0xc5, 0x3d, 0x5b, 0xa0, 0xe8, 0x62, 0x87, + 0xc8, 0xa8, 0x17, 0x62, 0x9b, 0x23, 0xec, 0x10, 0x24, 0x25, 0xf0, 0x5b, 0x60, 0xd6, 0xf5, 0x6c, + 0x72, 0xb0, 0x2f, 0x7d, 0x2f, 0x98, 0xf7, 0x94, 0xce, 0xec, 0x91, 0xdc, 0x45, 0x4a, 0x2a, 0x6e, + 0x91, 0x7b, 0xbe, 0xd7, 0xf6, 0x5a, 0xdd, 0x1f, 0x91, 0xee, 0xa0, 0xa4, 0xe5, 0x2d, 0x9e, 0x26, + 0xf6, 0x51, 0x4a, 0x0b, 0xfe, 0x1c, 0x94, 0x70, 0xbb, 0xed, 0x59, 0x98, 0xe3, 0xb3, 0x36, 0x91, + 0x75, 0x5a, 0xda, 0xf9, 0x62, 0xcc, 0xf1, 0xa2, 0x16, 0x10, 0x7e, 0x91, 0x22, 0x7e, 0x66, 0x2e, + 0xf7, 0x7b, 0xd5, 0xd2, 0x6e, 0x0c, 0x81, 0x92, 0x78, 0xfa, 0x9f, 0x34, 0x50, 0x52, 0x07, 0x9e, + 0x00, 0x5d, 0xed, 0xa5, 0xe9, 0xaa, 0x72, 0x73, 0x96, 0xc6, 0x90, 0xd5, 0x2f, 0x86, 0x11, 0x4b, + 0xa6, 0x3a, 0x06, 0x73, 0xb6, 0x4c, 0x15, 0x2b, 0x6b, 0x12, 0xf5, 0xb3, 0x9b, 0x51, 0x15, 0x11, + 0x2e, 0x2b, 0xec, 0xb9, 0x68, 0xcd, 0xd0, 0x00, 0x45, 0xff, 0x7f, 0x01, 0xc0, 0xbd, 0xe6, 0x41, + 0x86, 0x06, 0x26, 0x50, 0xc2, 0x14, 0x2c, 0x8a, 0x52, 0x19, 0x14, 0x83, 0x2a, 0xe5, 0x07, 0xb7, + 0xbc, 0x7f, 0x7c, 0x46, 0xda, 0x4d, 0xd2, 0x26, 0x16, 0xf7, 0x82, 0xa8, 0xaa, 0x8e, 0x12, 0x60, + 0x28, 0x05, 0x0d, 0xf7, 0xc1, 0xca, 0x80, 0xd5, 0xda, 0x98, 0x31, 0x51, 0xcd, 0xe5, 0x82, 0xac, + 0xde, 0xb2, 0x0a, 0x71, 0xa5, 0x99, 0x91, 0xa3, 0x11, 0x0b, 0xf8, 0x12, 0xcc, 0x5b, 0x49, 0x02, + 0xfd, 0x48, 0xb1, 0x18, 0x83, 0x69, 0xc4, 0xf8, 0x71, 0x88, 0x5d, 0x4e, 0x79, 0xd7, 0x5c, 0x14, + 0x85, 0x32, 0x64, 0xda, 0x21, 0x1a, 0x64, 0x60, 0xd5, 0xc1, 0x57, 0xd4, 0x09, 0x9d, 0xa8, 0xa4, + 0x9b, 0xf4, 0x57, 0x44, 0xd2, 0xec, 0xdd, 0x5d, 0x48, 0x9a, 0x6b, 0x64, 0xc1, 0xd0, 0x28, 0xbe, + 0xfe, 0x0f, 0x0d, 0x7c, 0x32, 0x9a, 0xf8, 0x09, 0xb4, 0xc5, 0x51, 0xba, 0x2d, 0x3e, 0x1f, 0x5f, + 0xc0, 0x99, 0xd8, 0xc6, 0x74, 0xc8, 0xef, 0x66, 0xc1, 0x62, 0x32, 0x7d, 0x13, 0xa8, 0xdd, 0xef, + 0x81, 0x92, 0x1f, 0x78, 0x1d, 0xca, 0xa8, 0xe7, 0x92, 0x40, 0x31, 0xe1, 0x9a, 0x32, 0x29, 0x9d, + 0xc4, 0x22, 0x94, 0xd4, 0x83, 0x2d, 0x00, 0x7c, 0x1c, 0x60, 0x87, 0x70, 0xd1, 0xbf, 0x05, 0x79, + 0xfc, 0x07, 0x63, 0x8e, 0x9f, 0x3c, 0x91, 0x71, 0x32, 0xb4, 0xaa, 0xbb, 0x3c, 0xe8, 0xc6, 0xd1, + 0xc5, 0x02, 0x94, 0x80, 0x86, 0x97, 0x60, 0x29, 0x20, 0x56, 0x1b, 0x53, 0x47, 0xbd, 0xd9, 0x45, + 0x19, 0x61, 0x5d, 0x3c, 0xa0, 0x28, 0x29, 0xb8, 0xee, 0x55, 0xbf, 0x1a, 0x9d, 0xba, 0x8d, 0x13, + 0x12, 0x30, 0xca, 0x38, 0x71, 0x79, 0x54, 0x30, 0x29, 0x1b, 0x94, 0xc6, 0x16, 0x4c, 0xef, 0x88, + 0x27, 0xf0, 0xd8, 0xe7, 0xd4, 0x73, 0x59, 0x79, 0x26, 0x66, 0xfa, 0x46, 0x62, 0x1f, 0xa5, 0xb4, + 0xe0, 0x21, 0x58, 0x17, 0xcc, 0xfc, 0xcb, 0xc8, 0x41, 0xfd, 0xca, 0xc7, 0xae, 0xb8, 0xa5, 0xf2, + 0xac, 0x7c, 0x6d, 0xcb, 0x62, 0xf4, 0xd9, 0xcd, 0x91, 0xa3, 0x5c, 0x2b, 0xf8, 0x12, 0xac, 0x46, + 0xb3, 0x8f, 0x49, 0x5d, 0x9b, 0xba, 0x2d, 0x31, 0xf9, 0xc8, 0x87, 0x7f, 0xc1, 0xfc, 0x42, 0x74, + 0xc4, 0x8b, 0xac, 0xf0, 0x3a, 0x6f, 0x13, 0x8d, 0x82, 0xc0, 0x37, 0x60, 0x55, 0x7a, 0x24, 0xb6, + 0xa2, 0x13, 0x4a, 0x58, 0x79, 0x5e, 0xa6, 0x6e, 0x3b, 0x99, 0x3a, 0x71, 0x75, 0xd1, 0xd4, 0x12, + 0x91, 0xce, 0x80, 0x9c, 0x4e, 0x49, 0xe0, 0x98, 0xdf, 0x54, 0xf9, 0x5a, 0xdd, 0xcd, 0x42, 0xa1, + 0x51, 0xf4, 0xcd, 0xa7, 0x60, 0x39, 0x93, 0x70, 0xb8, 0x02, 0x0a, 0x97, 0xa4, 0x1b, 0x3d, 0xcb, + 0x48, 0xfc, 0x84, 0xeb, 0x60, 0xa6, 0x83, 0xdb, 0x21, 0x89, 0x8a, 0x0f, 0x45, 0x8b, 0x27, 0xd3, + 0x8f, 0x35, 0xfd, 0x6f, 0x1a, 0x48, 0xd1, 0xd9, 0x04, 0x5a, 0xfa, 0x79, 0xba, 0xa5, 0x3f, 0xbd, + 0x45, 0x4d, 0x8f, 0x69, 0xe6, 0xdf, 0x68, 0x60, 0x31, 0x39, 0xe2, 0xc1, 0xef, 0x82, 0x79, 0x1c, + 0xda, 0x94, 0xb8, 0xd6, 0x60, 0x2a, 0x19, 0x06, 0xb2, 0xab, 0xf6, 0xd1, 0x50, 0x43, 0x0c, 0x80, + 0xe4, 0xca, 0xa7, 0x01, 0x16, 0x45, 0xd6, 0x24, 0x96, 0xe7, 0xda, 0x4c, 0xde, 0x50, 0x21, 0x62, + 0xc6, 0x7a, 0x56, 0x88, 0x46, 0xf5, 0xf5, 0x3f, 0x4e, 0x83, 0x95, 0xa8, 0x36, 0xa2, 0xd1, 0xdf, + 0x21, 0x2e, 0x9f, 0x00, 0xa9, 0x34, 0x52, 0x33, 0xdd, 0x77, 0x6e, 0x1c, 0x7a, 0xe2, 0xc0, 0xc6, + 0x0d, 0x77, 0xf0, 0x27, 0x60, 0x96, 0x71, 0xcc, 0x43, 0x26, 0x9f, 0xba, 0xd2, 0xce, 0x97, 0xb7, + 0x05, 0x94, 0x46, 0xf1, 0x5c, 0x17, 0xad, 0x91, 0x02, 0xd3, 0xff, 0xae, 0x81, 0xf5, 0xac, 0xc9, + 0x04, 0x2a, 0xec, 0x30, 0x5d, 0x61, 0xdf, 0xbe, 0xe5, 0x61, 0xc6, 0x7d, 0x01, 0x6a, 0xe0, 0x93, + 0x91, 0x73, 0xcb, 0x97, 0x54, 0xf0, 0x92, 0x9f, 0x61, 0xbf, 0xa3, 0x78, 0x22, 0x96, 0xbc, 0x74, + 0x92, 0x23, 0x47, 0xb9, 0x56, 0xf0, 0x35, 0x58, 0xa1, 0x6e, 0x9b, 0xba, 0x44, 0x3d, 0xbc, 0x71, + 0x7e, 0x73, 0xc9, 0x23, 0x8b, 0x2c, 0x93, 0xbb, 0x2e, 0xe6, 0x93, 0x83, 0x0c, 0x0a, 0x1a, 0xc1, + 0xd5, 0xff, 0x99, 0x93, 0x19, 0x39, 0x33, 0x8a, 0x16, 0x92, 0x3b, 0x24, 0x18, 0x69, 0x21, 0xb5, + 0x8f, 0x86, 0x1a, 0xb2, 0x6e, 0xe4, 0x55, 0xa8, 0x40, 0x6f, 0x5d, 0x37, 0xd2, 0x28, 0x51, 0x37, + 0x72, 0x8d, 0x14, 0x98, 0x08, 0x42, 0xcc, 0x64, 0x89, 0xd9, 0x6b, 0x18, 0xc4, 0x91, 0xda, 0x47, + 0x43, 0x0d, 0xfd, 0xbf, 0x85, 0x9c, 0x04, 0xc9, 0x02, 0x4c, 0x9c, 0x66, 0xf0, 0x95, 0x9e, 0x3d, + 0x8d, 0x3d, 0x3c, 0x8d, 0x0d, 0xff, 0xa0, 0x01, 0x88, 0x87, 0x10, 0x8d, 0x41, 0x81, 0x46, 0x55, + 0x54, 0xbf, 0x53, 0x4b, 0x18, 0xbb, 0x23, 0x38, 0xd1, 0x6b, 0xbc, 0xa9, 0xfc, 0xc3, 0x51, 0x05, + 0x94, 0xe3, 0x1c, 0xda, 0xa0, 0x14, 0xed, 0xd6, 0x83, 0xc0, 0x0b, 0x54, 0x7b, 0xea, 0x37, 0xc6, + 0x22, 0x35, 0xcd, 0x8a, 0xfc, 0xb8, 0x89, 0x4d, 0xaf, 0x7b, 0xd5, 0x52, 0x42, 0x8e, 0x92, 0xb0, + 0xc2, 0x8b, 0x4d, 0x62, 0x2f, 0xc5, 0xbb, 0x79, 0xd9, 0x27, 0xe3, 0xbd, 0x24, 0x60, 0x37, 0xeb, + 0xe0, 0x1b, 0x63, 0xae, 0xe5, 0x4e, 0x6f, 0xd6, 0x6f, 0x35, 0x90, 0xf4, 0x01, 0x0f, 0x41, 0x91, + 0x53, 0xd5, 0x75, 0xe9, 0x0f, 0xc0, 0x1b, 0x88, 0xe4, 0x94, 0x3a, 0x24, 0xa6, 0x42, 0xb1, 0x42, + 0x12, 0x05, 0x7e, 0x0e, 0xe6, 0x1c, 0xc2, 0x18, 0x6e, 0x29, 0xcf, 0xf1, 0xe7, 0x50, 0x23, 0xda, + 0x46, 0x03, 0xb9, 0xfe, 0x08, 0xac, 0xe5, 0x7c, 0x56, 0xc2, 0x2a, 0x98, 0xb1, 0xe4, 0x9f, 0x01, + 0x22, 0xa0, 0x19, 0x73, 0x41, 0x30, 0xca, 0x9e, 0xfc, 0x17, 0x20, 0xda, 0x37, 0xbf, 0xff, 0xf6, + 0x43, 0x65, 0xea, 0xdd, 0x87, 0xca, 0xd4, 0xfb, 0x0f, 0x95, 0xa9, 0x5f, 0xf7, 0x2b, 0xda, 0xdb, + 0x7e, 0x45, 0x7b, 0xd7, 0xaf, 0x68, 0xef, 0xfb, 0x15, 0xed, 0xdf, 0xfd, 0x8a, 0xf6, 0xfb, 0xff, + 0x54, 0xa6, 0x7e, 0xba, 0x91, 0xfb, 0x77, 0xea, 0xd7, 0x01, 0x00, 0x00, 0xff, 0xff, 0x0d, 0xf9, + 0xe3, 0xd5, 0x7f, 0x15, 0x00, 0x00, } func (m *CSIDriver) Marshal() (dAtA []byte, err error) { @@ -826,6 +827,16 @@ func (m *CSIDriverSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SELinuxMount != nil { + i-- + if *m.SELinuxMount { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x40 + } if m.RequiresRepublish != nil { i-- if *m.RequiresRepublish { @@ -1795,6 +1806,9 @@ func (m *CSIDriverSpec) Size() (n int) { if m.RequiresRepublish != nil { n += 2 } + if m.SELinuxMount != nil { + n += 2 + } return n } @@ -2148,6 +2162,7 @@ func (this *CSIDriverSpec) String() string { `FSGroupPolicy:` + valueToStringGenerated(this.FSGroupPolicy) + `,`, `TokenRequests:` + repeatedStringForTokenRequests + `,`, `RequiresRepublish:` + valueToStringGenerated(this.RequiresRepublish) + `,`, + `SELinuxMount:` + valueToStringGenerated(this.SELinuxMount) + `,`, `}`, }, "") return s @@ -2844,6 +2859,27 @@ func (m *CSIDriverSpec) Unmarshal(dAtA []byte) error { } b := bool(v != 0) m.RequiresRepublish = &b + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SELinuxMount", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + b := bool(v != 0) + m.SELinuxMount = &b default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/storage/v1/generated.proto b/staging/src/k8s.io/api/storage/v1/generated.proto index 45417116a1f..d3c425c0419 100644 --- a/staging/src/k8s.io/api/storage/v1/generated.proto +++ b/staging/src/k8s.io/api/storage/v1/generated.proto @@ -192,6 +192,27 @@ message CSIDriverSpec { // // +optional optional bool requiresRepublish = 7; + + // SELinuxMount specifies if the CSI driver supports "-o context" + // mount option. + // + // When "true", the CSI driver must ensure that all volumes provided by this CSI + // driver can be mounted separately with different `-o context` options. This is + // typical for storage backends that provide volumes as filesystems on block + // devices or as independent shared volumes. + // Kubernetes will call NodeStage / NodePublish with "-o context=xyz" mount + // option when mounting a ReadWriteOncePod volume used in Pod that has + // explicitly set SELinux context. In the future, it may be expanded to other + // volume AccessModes. In any case, Kubernetes will ensure that the volume is + // mounted only with a single SELinux context. + // + // When "false", Kubernetes won't pass any special SELinux mount options to the driver. + // This is typical for volumes that represent subdirectories of a bigger shared filesystem. + // + // Default is "false". + // + // +optional + optional bool seLinuxMount = 8; } // CSINode holds information about all CSI drivers installed on a node. diff --git a/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go index 7a407db46b2..1a069bb4037 100644 --- a/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/storage/v1/types_swagger_doc_generated.go @@ -56,6 +56,7 @@ var map_CSIDriverSpec = map[string]string{ "fsGroupPolicy": "Defines if the underlying volume supports changing ownership and permission of the volume before being mounted. Refer to the specific FSGroupPolicy values for additional details.\n\nThis field is immutable.\n\nDefaults to ReadWriteOnceWithFSType, which will examine each volume to determine if Kubernetes should modify ownership and permissions of the volume. With the default policy the defined fsGroup will only be applied if a fstype is defined and the volume's access mode contains ReadWriteOnce.", "tokenRequests": "TokenRequests indicates the CSI driver needs pods' service account tokens it is mounting volume for to do necessary authentication. Kubelet will pass the tokens in VolumeContext in the CSI NodePublishVolume calls. The CSI driver should parse and validate the following VolumeContext: \"csi.storage.k8s.io/serviceAccount.tokens\": {\n \"\": {\n \"token\": ,\n \"expirationTimestamp\": ,\n },\n ...\n}\n\nNote: Audience in each TokenRequest should be different and at most one token is empty string. To receive a new token after expiry, RequiresRepublish can be used to trigger NodePublishVolume periodically.", "requiresRepublish": "RequiresRepublish indicates the CSI driver wants `NodePublishVolume` being periodically called to reflect any possible change in the mounted volume. This field defaults to false.\n\nNote: After a successful initial NodePublishVolume call, subsequent calls to NodePublishVolume should only update the contents of the volume. New mount points will not be seen by a running container.", + "seLinuxMount": "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", } func (CSIDriverSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go index b14b7fbcd87..74ae83bca82 100644 --- a/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/storage/v1/zz_generated.deepcopy.go @@ -127,6 +127,11 @@ func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = new(bool) **out = **in } + if in.SELinuxMount != nil { + in, out := &in.SELinuxMount, &out.SELinuxMount + *out = new(bool) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go b/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go index 379ce8f1836..42ef65ca0fa 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go @@ -609,111 +609,112 @@ func init() { } var fileDescriptor_7d2980599fd0de80 = []byte{ - // 1654 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xcd, 0x6f, 0x1b, 0x37, - 0x16, 0xf7, 0x58, 0xf2, 0x17, 0x65, 0xc7, 0x36, 0xed, 0x64, 0xb5, 0x3a, 0x48, 0x86, 0x16, 0xbb, - 0x71, 0x82, 0xec, 0x28, 0xf1, 0x66, 0x83, 0x20, 0x40, 0x80, 0xf5, 0xd8, 0xde, 0x8d, 0x12, 0xcb, - 0x71, 0x28, 0x23, 0x08, 0x82, 0x3d, 0x2c, 0x35, 0x43, 0xcb, 0x8c, 0x35, 0x1f, 0x19, 0x52, 0xde, - 0xa8, 0xa7, 0xf6, 0xd2, 0x73, 0xd1, 0x43, 0xef, 0x05, 0xfa, 0x2f, 0xb4, 0x40, 0x7b, 0xe9, 0xb1, - 0x01, 0x0a, 0x14, 0x41, 0x4f, 0x39, 0x09, 0x8d, 0xfa, 0x27, 0x14, 0xe8, 0xc1, 0xe8, 0xa1, 0x20, - 0x87, 0xd2, 0x7c, 0x49, 0xb1, 0xdd, 0x83, 0x6e, 0xe2, 0xfb, 0xf8, 0xbd, 0x47, 0xf2, 0xf7, 0x1e, - 0xdf, 0x08, 0x6c, 0x1d, 0xdf, 0x65, 0x3a, 0x75, 0x2b, 0xc7, 0xed, 0x06, 0xf1, 0x1d, 0xc2, 0x09, - 0xab, 0x9c, 0x10, 0xc7, 0x72, 0xfd, 0x8a, 0x52, 0x60, 0x8f, 0x56, 0x18, 0x77, 0x7d, 0xdc, 0x24, - 0x95, 0x93, 0x5b, 0x0d, 0xc2, 0xf1, 0xad, 0x4a, 0x93, 0x38, 0xc4, 0xc7, 0x9c, 0x58, 0xba, 0xe7, - 0xbb, 0xdc, 0x85, 0x85, 0xc0, 0x56, 0xc7, 0x1e, 0xd5, 0x95, 0xad, 0xae, 0x6c, 0x0b, 0x7f, 0x6f, - 0x52, 0x7e, 0xd4, 0x6e, 0xe8, 0xa6, 0x6b, 0x57, 0x9a, 0x6e, 0xd3, 0xad, 0x48, 0x97, 0x46, 0xfb, - 0x50, 0xae, 0xe4, 0x42, 0xfe, 0x0a, 0xa0, 0x0a, 0xe5, 0x48, 0x58, 0xd3, 0xf5, 0x45, 0xcc, 0x64, - 0xb8, 0xc2, 0xed, 0xd0, 0xc6, 0xc6, 0xe6, 0x11, 0x75, 0x88, 0xdf, 0xa9, 0x78, 0xc7, 0x4d, 0xe9, - 0xe4, 0x13, 0xe6, 0xb6, 0x7d, 0x93, 0x5c, 0xc8, 0x8b, 0x55, 0x6c, 0xc2, 0xf1, 0xb0, 0x58, 0x95, - 0x51, 0x5e, 0x7e, 0xdb, 0xe1, 0xd4, 0x4e, 0x87, 0xb9, 0x73, 0x96, 0x03, 0x33, 0x8f, 0x88, 0x8d, - 0x93, 0x7e, 0xe5, 0x6f, 0x34, 0x30, 0xb7, 0x55, 0xaf, 0x6e, 0xfb, 0xf4, 0x84, 0xf8, 0xf0, 0x7f, - 0x60, 0x56, 0x64, 0x64, 0x61, 0x8e, 0xf3, 0xda, 0x9a, 0xb6, 0x9e, 0xdb, 0xb8, 0xa9, 0x87, 0x87, - 0x3c, 0x00, 0xd6, 0xbd, 0xe3, 0xa6, 0x10, 0x30, 0x5d, 0x58, 0xeb, 0x27, 0xb7, 0xf4, 0xc7, 0x8d, - 0x17, 0xc4, 0xe4, 0x35, 0xc2, 0xb1, 0x01, 0x5f, 0x77, 0x4b, 0x13, 0xbd, 0x6e, 0x09, 0x84, 0x32, - 0x34, 0x40, 0x85, 0x8f, 0x40, 0x96, 0x79, 0xc4, 0xcc, 0x4f, 0x4a, 0xf4, 0x6b, 0xfa, 0xe8, 0x2b, - 0xd4, 0x07, 0x69, 0xd5, 0x3d, 0x62, 0x1a, 0xf3, 0x0a, 0x36, 0x2b, 0x56, 0x48, 0x82, 0x94, 0xbf, - 0xd6, 0xc0, 0xc2, 0xc0, 0x6a, 0x97, 0x32, 0x0e, 0xff, 0x9b, 0xda, 0x80, 0x7e, 0xbe, 0x0d, 0x08, - 0x6f, 0x99, 0xfe, 0x92, 0x8a, 0x33, 0xdb, 0x97, 0x44, 0x92, 0x7f, 0x08, 0xa6, 0x28, 0x27, 0x36, - 0xcb, 0x4f, 0xae, 0x65, 0xd6, 0x73, 0x1b, 0x7f, 0x3d, 0x57, 0xf6, 0xc6, 0x82, 0x42, 0x9c, 0xaa, - 0x0a, 0x5f, 0x14, 0x40, 0x94, 0x3f, 0xcf, 0x46, 0x72, 0x17, 0x7b, 0x82, 0xf7, 0xc0, 0x25, 0xcc, - 0x39, 0x36, 0x8f, 0x10, 0x79, 0xd9, 0xa6, 0x3e, 0xb1, 0xe4, 0x0e, 0x66, 0x0d, 0xd8, 0xeb, 0x96, - 0x2e, 0x6d, 0xc6, 0x34, 0x28, 0x61, 0x29, 0x7c, 0x3d, 0xd7, 0xaa, 0x3a, 0x87, 0xee, 0x63, 0xa7, - 0xe6, 0xb6, 0x1d, 0x2e, 0x0f, 0x58, 0xf9, 0xee, 0xc7, 0x34, 0x28, 0x61, 0x09, 0x4d, 0xb0, 0x7a, - 0xe2, 0xb6, 0xda, 0x36, 0xd9, 0xa5, 0x87, 0xc4, 0xec, 0x98, 0x2d, 0x52, 0x73, 0x2d, 0xc2, 0xf2, - 0x99, 0xb5, 0xcc, 0xfa, 0x9c, 0x51, 0xe9, 0x75, 0x4b, 0xab, 0x4f, 0x87, 0xe8, 0x4f, 0xbb, 0xa5, - 0x95, 0x21, 0x72, 0x34, 0x14, 0x0c, 0xde, 0x07, 0x8b, 0xea, 0x84, 0xb6, 0xb0, 0x87, 0x4d, 0xca, - 0x3b, 0xf9, 0xac, 0xcc, 0x70, 0xa5, 0xd7, 0x2d, 0x2d, 0xd6, 0xe3, 0x2a, 0x94, 0xb4, 0x85, 0x0f, - 0xc0, 0xc2, 0x21, 0xfb, 0x8f, 0xef, 0xb6, 0xbd, 0x7d, 0xb7, 0x45, 0xcd, 0x4e, 0x7e, 0x6a, 0x4d, - 0x5b, 0x9f, 0x33, 0xca, 0xbd, 0x6e, 0x69, 0xe1, 0xdf, 0xf5, 0x88, 0xe2, 0x34, 0x29, 0x40, 0x71, - 0x47, 0x48, 0xc0, 0x02, 0x77, 0x8f, 0x89, 0x23, 0x8e, 0x8e, 0x30, 0xce, 0xf2, 0xd3, 0xf2, 0x2e, - 0xd7, 0xdf, 0x77, 0x97, 0x07, 0x11, 0x07, 0xe3, 0xb2, 0xba, 0xce, 0x85, 0xa8, 0x94, 0xa1, 0x38, - 0x2a, 0xdc, 0x02, 0xcb, 0x7e, 0x70, 0x39, 0x0c, 0x11, 0xaf, 0xdd, 0x68, 0x51, 0x76, 0x94, 0x9f, - 0x91, 0x3b, 0xbe, 0xdc, 0xeb, 0x96, 0x96, 0x51, 0x52, 0x89, 0xd2, 0xf6, 0xe5, 0xaf, 0x34, 0x30, - 0xb3, 0x55, 0xaf, 0xee, 0xb9, 0x16, 0x19, 0x43, 0x69, 0x56, 0x63, 0xa5, 0x79, 0xf5, 0x0c, 0x72, - 0x8b, 0xa4, 0x46, 0x16, 0xe6, 0x2f, 0x41, 0x61, 0x0a, 0x1b, 0xd5, 0x59, 0xd6, 0x40, 0xd6, 0xc1, - 0x36, 0x91, 0xa9, 0xcf, 0x85, 0x3e, 0x7b, 0xd8, 0x26, 0x48, 0x6a, 0xe0, 0xdf, 0xc0, 0xb4, 0xe3, - 0x5a, 0xa4, 0xba, 0x2d, 0x13, 0x98, 0x33, 0x2e, 0x29, 0x9b, 0xe9, 0x3d, 0x29, 0x45, 0x4a, 0x0b, - 0x6f, 0x83, 0x79, 0xee, 0x7a, 0x6e, 0xcb, 0x6d, 0x76, 0x1e, 0x91, 0x4e, 0x9f, 0xa6, 0x4b, 0xbd, - 0x6e, 0x69, 0xfe, 0x20, 0x22, 0x47, 0x31, 0x2b, 0xd8, 0x00, 0x39, 0xdc, 0x6a, 0xb9, 0x26, 0xe6, - 0xb8, 0xd1, 0x22, 0x92, 0x7b, 0xb9, 0x8d, 0xca, 0xfb, 0xf6, 0x18, 0x70, 0x5b, 0x04, 0x47, 0xaa, - 0xb7, 0x33, 0x63, 0xb1, 0xd7, 0x2d, 0xe5, 0x36, 0x43, 0x1c, 0x14, 0x05, 0x2d, 0x7f, 0xa9, 0x81, - 0x9c, 0xda, 0xf5, 0x18, 0x9a, 0xd1, 0x83, 0x78, 0x33, 0xfa, 0xcb, 0x39, 0xee, 0x6b, 0x44, 0x2b, - 0x32, 0x07, 0x69, 0xcb, 0x3e, 0x74, 0x00, 0x66, 0x2c, 0x79, 0x69, 0x2c, 0xaf, 0x49, 0xe8, 0x6b, - 0xe7, 0x80, 0x56, 0xbd, 0x6e, 0x51, 0x05, 0x98, 0x09, 0xd6, 0x0c, 0xf5, 0xa1, 0xca, 0xbf, 0x66, - 0x00, 0xdc, 0xaa, 0x57, 0x13, 0x95, 0x3e, 0x06, 0x5a, 0x53, 0x30, 0x2f, 0x98, 0xd3, 0xe7, 0x86, - 0xa2, 0xf7, 0x3f, 0xce, 0x79, 0x13, 0xb8, 0x41, 0x5a, 0x75, 0xd2, 0x22, 0x26, 0x77, 0xfd, 0x80, - 0x64, 0x7b, 0x11, 0x30, 0x14, 0x83, 0x86, 0xdb, 0x60, 0xa9, 0xdf, 0xb8, 0x5a, 0x98, 0x31, 0x41, - 0xee, 0x7c, 0x46, 0x92, 0x39, 0xaf, 0x52, 0x5c, 0xaa, 0x27, 0xf4, 0x28, 0xe5, 0x01, 0x9f, 0x81, - 0x59, 0x33, 0xda, 0x23, 0xcf, 0xa0, 0x8d, 0xde, 0x1f, 0x3d, 0xf4, 0x27, 0x6d, 0xec, 0x70, 0xca, - 0x3b, 0xc6, 0xbc, 0xa0, 0xcc, 0xa0, 0x99, 0x0e, 0xd0, 0x20, 0x03, 0xcb, 0x36, 0x7e, 0x45, 0xed, - 0xb6, 0x1d, 0x90, 0xbb, 0x4e, 0x3f, 0x20, 0xb2, 0x93, 0x5e, 0x3c, 0x84, 0x6c, 0x62, 0xb5, 0x24, - 0x18, 0x4a, 0xe3, 0x97, 0xbf, 0xd7, 0xc0, 0x95, 0xf4, 0xc5, 0x8f, 0xa1, 0x40, 0xea, 0xf1, 0x02, - 0xd1, 0xcf, 0x60, 0x71, 0x22, 0xc1, 0x11, 0xb5, 0xf2, 0xe9, 0x34, 0x98, 0x8f, 0xde, 0xe1, 0x18, - 0x08, 0xfc, 0x4f, 0x90, 0xf3, 0x7c, 0xf7, 0x84, 0x32, 0xea, 0x3a, 0xc4, 0x57, 0xdd, 0x71, 0x45, - 0xb9, 0xe4, 0xf6, 0x43, 0x15, 0x8a, 0xda, 0xc1, 0x16, 0x00, 0x1e, 0xf6, 0xb1, 0x4d, 0xb8, 0xa8, - 0xe4, 0x8c, 0x3c, 0x83, 0xbb, 0xef, 0x3b, 0x83, 0xe8, 0xb6, 0xf4, 0xfd, 0x81, 0xeb, 0x8e, 0xc3, - 0xfd, 0x4e, 0x98, 0x62, 0xa8, 0x40, 0x11, 0x7c, 0x78, 0x0c, 0x16, 0x7c, 0x62, 0xb6, 0x30, 0xb5, - 0xd5, 0x03, 0x9d, 0x95, 0x69, 0xee, 0x88, 0x87, 0x12, 0x45, 0x15, 0xa7, 0xdd, 0xd2, 0xcd, 0xf4, - 0xb0, 0xad, 0xef, 0x13, 0x9f, 0x51, 0xc6, 0x89, 0xc3, 0x03, 0xea, 0xc4, 0x7c, 0x50, 0x1c, 0x5b, - 0x3c, 0x01, 0xb6, 0x18, 0x5d, 0x1e, 0x7b, 0x9c, 0xba, 0x0e, 0xcb, 0x4f, 0x85, 0x4f, 0x40, 0x2d, - 0x22, 0x47, 0x31, 0x2b, 0xb8, 0x0b, 0x56, 0x45, 0xb7, 0xfe, 0x7f, 0x10, 0x60, 0xe7, 0x95, 0x87, - 0x1d, 0x71, 0x54, 0xf9, 0x69, 0xf9, 0x2a, 0xe7, 0xc5, 0x9c, 0xb3, 0x39, 0x44, 0x8f, 0x86, 0x7a, - 0xc1, 0x67, 0x60, 0x39, 0x18, 0x74, 0x0c, 0xea, 0x58, 0xd4, 0x69, 0x8a, 0x31, 0x47, 0x3e, 0xf0, - 0x73, 0xc6, 0x75, 0x51, 0x1b, 0x4f, 0x93, 0xca, 0xd3, 0x61, 0x42, 0x94, 0x06, 0x81, 0x2f, 0xc1, - 0xb2, 0x8c, 0x48, 0x2c, 0xd5, 0x58, 0x28, 0x61, 0xf9, 0xd9, 0xf4, 0x94, 0x22, 0x8e, 0x4e, 0x10, - 0xa9, 0xdf, 0x7e, 0xfa, 0x6d, 0xea, 0x80, 0xf8, 0xb6, 0xf1, 0x67, 0x75, 0x5f, 0xcb, 0x9b, 0x49, - 0x28, 0x94, 0x46, 0x2f, 0xdc, 0x07, 0x8b, 0x89, 0x0b, 0x87, 0x4b, 0x20, 0x73, 0x4c, 0x3a, 0xc1, - 0x7b, 0x8d, 0xc4, 0x4f, 0xb8, 0x0a, 0xa6, 0x4e, 0x70, 0xab, 0x4d, 0x02, 0x06, 0xa2, 0x60, 0x71, - 0x6f, 0xf2, 0xae, 0x56, 0xfe, 0x56, 0x03, 0xb1, 0xc6, 0x36, 0x86, 0xe2, 0xae, 0xc5, 0x8b, 0x7b, - 0xfd, 0xbc, 0xc4, 0x1e, 0x51, 0xd6, 0x1f, 0x69, 0x60, 0x3e, 0x3a, 0xcf, 0xc1, 0x1b, 0x60, 0x16, - 0xb7, 0x2d, 0x4a, 0x1c, 0xb3, 0x3f, 0xb3, 0x0c, 0xb2, 0xd9, 0x54, 0x72, 0x34, 0xb0, 0x10, 0xd3, - 0x1e, 0x79, 0xe5, 0x51, 0x1f, 0x0b, 0xa6, 0xd5, 0x89, 0xe9, 0x3a, 0x16, 0x93, 0xc7, 0x94, 0x09, - 0x1a, 0xe5, 0x4e, 0x52, 0x89, 0xd2, 0xf6, 0xe5, 0x2f, 0x26, 0xc1, 0x52, 0x40, 0x90, 0x60, 0xd8, - 0xb7, 0x89, 0xc3, 0xc7, 0xd0, 0x5e, 0x50, 0x6c, 0xec, 0xbb, 0x79, 0xf6, 0x48, 0x14, 0x66, 0x37, - 0x6a, 0xfe, 0x83, 0xcf, 0xc1, 0x34, 0xe3, 0x98, 0xb7, 0x99, 0x7c, 0xfe, 0x72, 0x1b, 0x1b, 0x17, - 0x42, 0x95, 0x9e, 0xe1, 0xfc, 0x17, 0xac, 0x91, 0x42, 0x2c, 0x7f, 0xa7, 0x81, 0xd5, 0xa4, 0xcb, - 0x18, 0x08, 0xf7, 0x24, 0x4e, 0xb8, 0x1b, 0x17, 0xd9, 0xd1, 0x08, 0xd2, 0xfd, 0xa8, 0x81, 0x2b, - 0xa9, 0xcd, 0xcb, 0x77, 0x56, 0xf4, 0x2a, 0x2f, 0xd1, 0x11, 0xf7, 0xc2, 0xf1, 0x59, 0xf6, 0xaa, - 0xfd, 0x21, 0x7a, 0x34, 0xd4, 0x0b, 0xbe, 0x00, 0x4b, 0xd4, 0x69, 0x51, 0x87, 0xa8, 0x67, 0x39, - 0xbc, 0xee, 0xa1, 0x0d, 0x25, 0x89, 0x2c, 0xaf, 0x79, 0x55, 0x4c, 0x2f, 0xd5, 0x04, 0x0a, 0x4a, - 0xe1, 0x96, 0x7f, 0x18, 0x72, 0x3d, 0x72, 0xac, 0x14, 0x15, 0x25, 0x25, 0xc4, 0x4f, 0x55, 0x94, - 0x92, 0xa3, 0x81, 0x85, 0x64, 0x90, 0x3c, 0x0a, 0x95, 0xe8, 0xc5, 0x18, 0x24, 0x3d, 0x23, 0x0c, - 0x92, 0x6b, 0xa4, 0x10, 0x45, 0x26, 0x62, 0x6c, 0x8b, 0x8c, 0x67, 0x83, 0x4c, 0xf6, 0x94, 0x1c, - 0x0d, 0x2c, 0xca, 0xbf, 0x65, 0x86, 0xdc, 0x92, 0xa4, 0x62, 0x64, 0x4b, 0xfd, 0x6f, 0xf5, 0xe4, - 0x96, 0xac, 0xc1, 0x96, 0x2c, 0xf8, 0x99, 0x06, 0x20, 0x1e, 0x40, 0xd4, 0xfa, 0x54, 0x0d, 0xf8, - 0xf4, 0xf0, 0xe2, 0x15, 0xa2, 0x6f, 0xa6, 0xc0, 0x82, 0xb7, 0xba, 0xa0, 0x92, 0x80, 0x69, 0x03, - 0x34, 0x24, 0x03, 0x48, 0x41, 0x2e, 0x90, 0xee, 0xf8, 0xbe, 0xeb, 0xab, 0x92, 0xbd, 0x7a, 0x76, - 0x42, 0xd2, 0xdc, 0x28, 0xca, 0x6f, 0xa2, 0xd0, 0xff, 0xb4, 0x5b, 0xca, 0x45, 0xf4, 0x28, 0x8a, - 0x2d, 0x42, 0x59, 0x24, 0x0c, 0x95, 0xfd, 0x03, 0xa1, 0xb6, 0xc9, 0xe8, 0x50, 0x11, 0xec, 0xc2, - 0x0e, 0xf8, 0xd3, 0x88, 0x03, 0xba, 0xd0, 0xdb, 0xf6, 0xb1, 0x06, 0xa2, 0x31, 0xe0, 0x2e, 0xc8, - 0x72, 0xaa, 0x2a, 0x31, 0xb7, 0x71, 0xfd, 0x7c, 0x1d, 0xe6, 0x80, 0xda, 0x24, 0x6c, 0x94, 0x62, - 0x85, 0x24, 0x0a, 0xbc, 0x06, 0x66, 0x6c, 0xc2, 0x18, 0x6e, 0xaa, 0xc8, 0xe1, 0x07, 0x54, 0x2d, - 0x10, 0xa3, 0xbe, 0xbe, 0x7c, 0x07, 0xac, 0x0c, 0xf9, 0x24, 0x85, 0x25, 0x30, 0x65, 0xca, 0x3f, - 0x7c, 0x44, 0x42, 0x53, 0xc6, 0x9c, 0xe8, 0x32, 0x5b, 0xf2, 0x7f, 0x9e, 0x40, 0x6e, 0xfc, 0xeb, - 0xf5, 0xbb, 0xe2, 0xc4, 0x9b, 0x77, 0xc5, 0x89, 0xb7, 0xef, 0x8a, 0x13, 0x1f, 0xf6, 0x8a, 0xda, - 0xeb, 0x5e, 0x51, 0x7b, 0xd3, 0x2b, 0x6a, 0x6f, 0x7b, 0x45, 0xed, 0xa7, 0x5e, 0x51, 0xfb, 0xe4, - 0xe7, 0xe2, 0xc4, 0xf3, 0xc2, 0xe8, 0xff, 0x5d, 0x7f, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x91, 0x4f, - 0x2f, 0xc0, 0xad, 0x15, 0x00, 0x00, + // 1672 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xcd, 0x6f, 0x1b, 0x4d, + 0x19, 0xcf, 0xc6, 0xce, 0xd7, 0x38, 0x69, 0x92, 0x49, 0x5a, 0x8c, 0x0f, 0x76, 0x64, 0x04, 0x4d, + 0xab, 0xb2, 0x6e, 0x43, 0xa9, 0xaa, 0x4a, 0x95, 0xc8, 0x26, 0x81, 0xba, 0x8d, 0xd3, 0x74, 0x1c, + 0x55, 0x55, 0xc5, 0x81, 0xf1, 0xee, 0xc4, 0x99, 0xc6, 0xfb, 0xd1, 0x9d, 0xd9, 0x10, 0x73, 0x82, + 0x0b, 0x67, 0xc4, 0x81, 0xbf, 0x80, 0x7f, 0x01, 0x24, 0xb8, 0x70, 0xa4, 0x12, 0x12, 0xaa, 0xb8, + 0xd0, 0x93, 0x45, 0xcd, 0x9f, 0xf0, 0x4a, 0xef, 0x21, 0x7a, 0x0f, 0xaf, 0x66, 0x76, 0xec, 0xfd, + 0xb2, 0x9b, 0xe4, 0x3d, 0xf8, 0xe6, 0x79, 0x3e, 0x7e, 0xcf, 0x33, 0xf3, 0x7c, 0xae, 0xc1, 0xce, + 0xe9, 0x63, 0xa6, 0x53, 0xb7, 0x76, 0x1a, 0xb4, 0x88, 0xef, 0x10, 0x4e, 0x58, 0xed, 0x8c, 0x38, + 0x96, 0xeb, 0xd7, 0x14, 0x03, 0x7b, 0xb4, 0xc6, 0xb8, 0xeb, 0xe3, 0x36, 0xa9, 0x9d, 0x3d, 0x68, + 0x11, 0x8e, 0x1f, 0xd4, 0xda, 0xc4, 0x21, 0x3e, 0xe6, 0xc4, 0xd2, 0x3d, 0xdf, 0xe5, 0x2e, 0x2c, + 0x85, 0xb2, 0x3a, 0xf6, 0xa8, 0xae, 0x64, 0x75, 0x25, 0x5b, 0xfa, 0x71, 0x9b, 0xf2, 0x93, 0xa0, + 0xa5, 0x9b, 0xae, 0x5d, 0x6b, 0xbb, 0x6d, 0xb7, 0x26, 0x55, 0x5a, 0xc1, 0xb1, 0x3c, 0xc9, 0x83, + 0xfc, 0x15, 0x42, 0x95, 0xaa, 0x31, 0xb3, 0xa6, 0xeb, 0x0b, 0x9b, 0x69, 0x73, 0xa5, 0x87, 0x91, + 0x8c, 0x8d, 0xcd, 0x13, 0xea, 0x10, 0xbf, 0x5b, 0xf3, 0x4e, 0xdb, 0x52, 0xc9, 0x27, 0xcc, 0x0d, + 0x7c, 0x93, 0x5c, 0x4b, 0x8b, 0xd5, 0x6c, 0xc2, 0xf1, 0x28, 0x5b, 0xb5, 0x71, 0x5a, 0x7e, 0xe0, + 0x70, 0x6a, 0x67, 0xcd, 0x3c, 0xba, 0x4c, 0x81, 0x99, 0x27, 0xc4, 0xc6, 0x69, 0xbd, 0xea, 0xdf, + 0x35, 0xb0, 0xb0, 0xd3, 0xac, 0xef, 0xfa, 0xf4, 0x8c, 0xf8, 0xf0, 0x57, 0x60, 0x5e, 0x78, 0x64, + 0x61, 0x8e, 0x8b, 0xda, 0x86, 0xb6, 0x59, 0xd8, 0xba, 0xaf, 0x47, 0x8f, 0x3c, 0x04, 0xd6, 0xbd, + 0xd3, 0xb6, 0x20, 0x30, 0x5d, 0x48, 0xeb, 0x67, 0x0f, 0xf4, 0x97, 0xad, 0x77, 0xc4, 0xe4, 0x0d, + 0xc2, 0xb1, 0x01, 0x3f, 0xf4, 0x2a, 0x53, 0xfd, 0x5e, 0x05, 0x44, 0x34, 0x34, 0x44, 0x85, 0x2f, + 0x40, 0x9e, 0x79, 0xc4, 0x2c, 0x4e, 0x4b, 0xf4, 0x3b, 0xfa, 0xf8, 0x10, 0xea, 0x43, 0xb7, 0x9a, + 0x1e, 0x31, 0x8d, 0x45, 0x05, 0x9b, 0x17, 0x27, 0x24, 0x41, 0xaa, 0x7f, 0xd3, 0xc0, 0xd2, 0x50, + 0x6a, 0x9f, 0x32, 0x0e, 0x7f, 0x99, 0xb9, 0x80, 0x7e, 0xb5, 0x0b, 0x08, 0x6d, 0xe9, 0xfe, 0x8a, + 0xb2, 0x33, 0x3f, 0xa0, 0xc4, 0x9c, 0x7f, 0x0e, 0x66, 0x28, 0x27, 0x36, 0x2b, 0x4e, 0x6f, 0xe4, + 0x36, 0x0b, 0x5b, 0x3f, 0xbc, 0x92, 0xf7, 0xc6, 0x92, 0x42, 0x9c, 0xa9, 0x0b, 0x5d, 0x14, 0x42, + 0x54, 0xff, 0x9b, 0x8f, 0xf9, 0x2e, 0xee, 0x04, 0x9f, 0x80, 0x1b, 0x98, 0x73, 0x6c, 0x9e, 0x20, + 0xf2, 0x3e, 0xa0, 0x3e, 0xb1, 0xe4, 0x0d, 0xe6, 0x0d, 0xd8, 0xef, 0x55, 0x6e, 0x6c, 0x27, 0x38, + 0x28, 0x25, 0x29, 0x74, 0x3d, 0xd7, 0xaa, 0x3b, 0xc7, 0xee, 0x4b, 0xa7, 0xe1, 0x06, 0x0e, 0x97, + 0x0f, 0xac, 0x74, 0x0f, 0x13, 0x1c, 0x94, 0x92, 0x84, 0x26, 0x58, 0x3f, 0x73, 0x3b, 0x81, 0x4d, + 0xf6, 0xe9, 0x31, 0x31, 0xbb, 0x66, 0x87, 0x34, 0x5c, 0x8b, 0xb0, 0x62, 0x6e, 0x23, 0xb7, 0xb9, + 0x60, 0xd4, 0xfa, 0xbd, 0xca, 0xfa, 0xeb, 0x11, 0xfc, 0x8b, 0x5e, 0x65, 0x6d, 0x04, 0x1d, 0x8d, + 0x04, 0x83, 0x4f, 0xc1, 0xb2, 0x7a, 0xa1, 0x1d, 0xec, 0x61, 0x93, 0xf2, 0x6e, 0x31, 0x2f, 0x3d, + 0x5c, 0xeb, 0xf7, 0x2a, 0xcb, 0xcd, 0x24, 0x0b, 0xa5, 0x65, 0xe1, 0x33, 0xb0, 0x74, 0xcc, 0x7e, + 0xe1, 0xbb, 0x81, 0x77, 0xe8, 0x76, 0xa8, 0xd9, 0x2d, 0xce, 0x6c, 0x68, 0x9b, 0x0b, 0x46, 0xb5, + 0xdf, 0xab, 0x2c, 0xfd, 0xbc, 0x19, 0x63, 0x5c, 0xa4, 0x09, 0x28, 0xa9, 0x08, 0x09, 0x58, 0xe2, + 0xee, 0x29, 0x71, 0xc4, 0xd3, 0x11, 0xc6, 0x59, 0x71, 0x56, 0xc6, 0x72, 0xf3, 0x4b, 0xb1, 0x3c, + 0x8a, 0x29, 0x18, 0x37, 0x55, 0x38, 0x97, 0xe2, 0x54, 0x86, 0x92, 0xa8, 0x70, 0x07, 0xac, 0xfa, + 0x61, 0x70, 0x18, 0x22, 0x5e, 0xd0, 0xea, 0x50, 0x76, 0x52, 0x9c, 0x93, 0x37, 0xbe, 0xd9, 0xef, + 0x55, 0x56, 0x51, 0x9a, 0x89, 0xb2, 0xf2, 0xf0, 0x21, 0x58, 0x64, 0x64, 0x9f, 0x3a, 0xc1, 0x79, + 0x18, 0xd3, 0x79, 0xa9, 0xbf, 0xd2, 0xef, 0x55, 0x16, 0x9b, 0x7b, 0x11, 0x1d, 0x25, 0xa4, 0xaa, + 0x7f, 0xd5, 0xc0, 0xdc, 0x4e, 0xb3, 0x7e, 0xe0, 0x5a, 0x64, 0x02, 0x05, 0x5d, 0x4f, 0x14, 0xf4, + 0xed, 0x4b, 0x4a, 0x42, 0x38, 0x35, 0xb6, 0x9c, 0xbf, 0x0a, 0xcb, 0x59, 0xc8, 0xa8, 0x7e, 0xb4, + 0x01, 0xf2, 0x0e, 0xb6, 0x89, 0x74, 0x7d, 0x21, 0xd2, 0x39, 0xc0, 0x36, 0x41, 0x92, 0x03, 0x7f, + 0x04, 0x66, 0x1d, 0xd7, 0x22, 0xf5, 0x5d, 0xe9, 0xc0, 0x82, 0x71, 0x43, 0xc9, 0xcc, 0x1e, 0x48, + 0x2a, 0x52, 0x5c, 0xf1, 0x94, 0xdc, 0xf5, 0xdc, 0x8e, 0xdb, 0xee, 0xbe, 0x20, 0xdd, 0x41, 0x72, + 0xcb, 0xa7, 0x3c, 0x8a, 0xd1, 0x51, 0x42, 0x0a, 0xb6, 0x40, 0x01, 0x77, 0x3a, 0xae, 0x89, 0x39, + 0x6e, 0x75, 0x88, 0xcc, 0xd8, 0xc2, 0x56, 0xed, 0x4b, 0x77, 0x0c, 0x2b, 0x42, 0x18, 0x47, 0x6a, + 0x22, 0x30, 0x63, 0xb9, 0xdf, 0xab, 0x14, 0xb6, 0x23, 0x1c, 0x14, 0x07, 0xad, 0xfe, 0x45, 0x03, + 0x05, 0x75, 0xeb, 0x09, 0xb4, 0xb0, 0x67, 0xc9, 0x16, 0xf6, 0x83, 0x2b, 0xc4, 0x6b, 0x4c, 0x03, + 0x33, 0x87, 0x6e, 0xcb, 0xee, 0x75, 0x04, 0xe6, 0x2c, 0x19, 0x34, 0x56, 0xd4, 0x24, 0xf4, 0x9d, + 0x2b, 0x40, 0xab, 0x0e, 0xb9, 0xac, 0x0c, 0xcc, 0x85, 0x67, 0x86, 0x06, 0x50, 0xd5, 0xaf, 0x73, + 0x00, 0xee, 0x34, 0xeb, 0xa9, 0xfe, 0x30, 0x81, 0xb4, 0xa6, 0x60, 0x51, 0x64, 0xce, 0x20, 0x37, + 0x54, 0x7a, 0xff, 0xe4, 0x8a, 0x91, 0xc0, 0x2d, 0xd2, 0x69, 0x92, 0x0e, 0x31, 0xb9, 0xeb, 0x87, + 0x49, 0x76, 0x10, 0x03, 0x43, 0x09, 0x68, 0xb8, 0x0b, 0x56, 0x06, 0xed, 0xae, 0x83, 0x19, 0x13, + 0xc9, 0x5d, 0xcc, 0xc9, 0x64, 0x2e, 0x2a, 0x17, 0x57, 0x9a, 0x29, 0x3e, 0xca, 0x68, 0xc0, 0x37, + 0x60, 0xde, 0x8c, 0x77, 0xd6, 0x4b, 0xd2, 0x46, 0x1f, 0x2c, 0x2c, 0xfa, 0xab, 0x00, 0x3b, 0x9c, + 0xf2, 0xae, 0xb1, 0x28, 0x52, 0x66, 0xd8, 0x82, 0x87, 0x68, 0x90, 0x81, 0x55, 0x1b, 0x9f, 0x53, + 0x3b, 0xb0, 0xc3, 0xe4, 0x6e, 0xd2, 0xdf, 0x10, 0xd9, 0x7f, 0xaf, 0x6f, 0x42, 0xb6, 0xbe, 0x46, + 0x1a, 0x0c, 0x65, 0xf1, 0xab, 0xff, 0xd2, 0xc0, 0xad, 0x6c, 0xe0, 0x27, 0x50, 0x20, 0xcd, 0x64, + 0x81, 0xe8, 0x97, 0x64, 0x71, 0xca, 0xc1, 0x31, 0xb5, 0xf2, 0xc7, 0x59, 0xb0, 0x18, 0x8f, 0xe1, + 0x04, 0x12, 0xf8, 0xa7, 0xa0, 0xe0, 0xf9, 0xee, 0x19, 0x65, 0xd4, 0x75, 0x88, 0xaf, 0xba, 0xe3, + 0x9a, 0x52, 0x29, 0x1c, 0x46, 0x2c, 0x14, 0x97, 0x83, 0x1d, 0x00, 0x3c, 0xec, 0x63, 0x9b, 0x70, + 0x51, 0xc9, 0x39, 0xf9, 0x06, 0x8f, 0xbf, 0xf4, 0x06, 0xf1, 0x6b, 0xe9, 0x87, 0x43, 0xd5, 0x3d, + 0x87, 0xfb, 0xdd, 0xc8, 0xc5, 0x88, 0x81, 0x62, 0xf8, 0xf0, 0x14, 0x2c, 0xf9, 0xc4, 0xec, 0x60, + 0x6a, 0xab, 0xb1, 0x9e, 0x97, 0x6e, 0xee, 0x89, 0xf1, 0x8a, 0xe2, 0x8c, 0x8b, 0x5e, 0xe5, 0x7e, + 0x76, 0x45, 0xd7, 0x0f, 0x89, 0xcf, 0x28, 0xe3, 0xc4, 0xe1, 0x61, 0xea, 0x24, 0x74, 0x50, 0x12, + 0x5b, 0x8c, 0x00, 0x5b, 0x0c, 0xc8, 0x97, 0x1e, 0xa7, 0xae, 0xc3, 0x8a, 0x33, 0xd1, 0x08, 0x68, + 0xc4, 0xe8, 0x28, 0x21, 0x05, 0xf7, 0xc1, 0xba, 0xe8, 0xd6, 0xbf, 0x0e, 0x0d, 0xec, 0x9d, 0x7b, + 0xd8, 0x11, 0x4f, 0x55, 0x9c, 0x95, 0xb3, 0xb8, 0x28, 0xb6, 0xa3, 0xed, 0x11, 0x7c, 0x34, 0x52, + 0x0b, 0xbe, 0x01, 0xab, 0xe1, 0x7a, 0x64, 0x50, 0xc7, 0xa2, 0x4e, 0x5b, 0x2c, 0x47, 0x72, 0x2d, + 0x58, 0x30, 0xee, 0x8a, 0xda, 0x78, 0x9d, 0x66, 0x5e, 0x8c, 0x22, 0xa2, 0x2c, 0x08, 0x7c, 0x0f, + 0x56, 0xa5, 0x45, 0x62, 0xa9, 0xc6, 0x42, 0x09, 0x2b, 0xce, 0x67, 0x77, 0x1b, 0xf1, 0x74, 0x22, + 0x91, 0x06, 0xed, 0x67, 0xd0, 0xa6, 0x8e, 0x88, 0x6f, 0x1b, 0xdf, 0x57, 0xf1, 0x5a, 0xdd, 0x4e, + 0x43, 0xa1, 0x2c, 0x7a, 0xe9, 0x29, 0x58, 0x4e, 0x05, 0x1c, 0xae, 0x80, 0xdc, 0x29, 0xe9, 0x86, + 0xf3, 0x1a, 0x89, 0x9f, 0x70, 0x1d, 0xcc, 0x9c, 0xe1, 0x4e, 0x40, 0xc2, 0x0c, 0x44, 0xe1, 0xe1, + 0xc9, 0xf4, 0x63, 0xad, 0xfa, 0x0f, 0x0d, 0x24, 0x1a, 0xdb, 0x04, 0x8a, 0xbb, 0x91, 0x2c, 0xee, + 0xcd, 0xab, 0x26, 0xf6, 0x98, 0xb2, 0xfe, 0x9d, 0x06, 0x16, 0xe3, 0x5b, 0x20, 0xbc, 0x07, 0xe6, + 0x71, 0x60, 0x51, 0xe2, 0x98, 0x83, 0x9d, 0x65, 0xe8, 0xcd, 0xb6, 0xa2, 0xa3, 0xa1, 0x84, 0xd8, + 0x11, 0xc9, 0xb9, 0x47, 0x7d, 0x2c, 0x32, 0xad, 0x49, 0x4c, 0xd7, 0xb1, 0x98, 0x7c, 0xa6, 0x5c, + 0xd8, 0x28, 0xf7, 0xd2, 0x4c, 0x94, 0x95, 0xaf, 0xfe, 0x79, 0x1a, 0xac, 0x84, 0x09, 0x12, 0x7e, + 0x22, 0xd8, 0xc4, 0xe1, 0x13, 0x68, 0x2f, 0x28, 0xb1, 0xf6, 0xdd, 0xbf, 0x7c, 0x25, 0x8a, 0xbc, + 0x1b, 0xb7, 0xff, 0xc1, 0xb7, 0x60, 0x96, 0x71, 0xcc, 0x03, 0x26, 0xc7, 0x5f, 0x61, 0x6b, 0xeb, + 0x5a, 0xa8, 0x52, 0x33, 0xda, 0xff, 0xc2, 0x33, 0x52, 0x88, 0xd5, 0x7f, 0x6a, 0x60, 0x3d, 0xad, + 0x32, 0x81, 0x84, 0x7b, 0x95, 0x4c, 0xb8, 0x7b, 0xd7, 0xb9, 0xd1, 0x98, 0xa4, 0xfb, 0x8f, 0x06, + 0x6e, 0x65, 0x2e, 0x2f, 0xe7, 0xac, 0xe8, 0x55, 0x5e, 0xaa, 0x23, 0x1e, 0x44, 0xeb, 0xb3, 0xec, + 0x55, 0x87, 0x23, 0xf8, 0x68, 0xa4, 0x16, 0x7c, 0x07, 0x56, 0xa8, 0xd3, 0xa1, 0x0e, 0x51, 0x63, + 0x39, 0x0a, 0xf7, 0xc8, 0x86, 0x92, 0x46, 0x96, 0x61, 0x5e, 0x17, 0xdb, 0x4b, 0x3d, 0x85, 0x82, + 0x32, 0xb8, 0xd5, 0x7f, 0x8f, 0x08, 0x8f, 0x5c, 0x2b, 0x45, 0x45, 0x49, 0x0a, 0xf1, 0x33, 0x15, + 0xa5, 0xe8, 0x68, 0x28, 0x21, 0x33, 0x48, 0x3e, 0x85, 0x72, 0xf4, 0x7a, 0x19, 0x24, 0x35, 0x63, + 0x19, 0x24, 0xcf, 0x48, 0x21, 0x0a, 0x4f, 0xc4, 0xda, 0x16, 0x5b, 0xcf, 0x86, 0x9e, 0x1c, 0x28, + 0x3a, 0x1a, 0x4a, 0x54, 0xbf, 0xc9, 0x8d, 0x88, 0x92, 0x4c, 0xc5, 0xd8, 0x95, 0x06, 0x5f, 0xf8, + 0xe9, 0x2b, 0x59, 0xc3, 0x2b, 0x59, 0xf0, 0x4f, 0x1a, 0x80, 0x78, 0x08, 0xd1, 0x18, 0xa4, 0x6a, + 0x98, 0x4f, 0xcf, 0xaf, 0x5f, 0x21, 0xfa, 0x76, 0x06, 0x2c, 0x9c, 0xd5, 0x25, 0xe5, 0x04, 0xcc, + 0x0a, 0xa0, 0x11, 0x1e, 0x40, 0x0a, 0x0a, 0x21, 0x75, 0xcf, 0xf7, 0x5d, 0x5f, 0x95, 0xec, 0xed, + 0xcb, 0x1d, 0x92, 0xe2, 0x46, 0x59, 0x7e, 0x13, 0x45, 0xfa, 0x17, 0xbd, 0x4a, 0x21, 0xc6, 0x47, + 0x71, 0x6c, 0x61, 0xca, 0x22, 0x91, 0xa9, 0xfc, 0x77, 0x30, 0xb5, 0x4b, 0xc6, 0x9b, 0x8a, 0x61, + 0x97, 0xf6, 0xc0, 0xf7, 0xc6, 0x3c, 0xd0, 0xb5, 0x66, 0xdb, 0xef, 0x35, 0x10, 0xb7, 0x01, 0xf7, + 0x41, 0x9e, 0x53, 0x55, 0x89, 0x85, 0xad, 0xbb, 0x57, 0xeb, 0x30, 0x47, 0xd4, 0x26, 0x51, 0xa3, + 0x14, 0x27, 0x24, 0x51, 0xe0, 0x1d, 0x30, 0x67, 0x13, 0xc6, 0x70, 0x5b, 0x59, 0x8e, 0x3e, 0xa0, + 0x1a, 0x21, 0x19, 0x0d, 0xf8, 0xd5, 0x47, 0x60, 0x6d, 0xc4, 0x27, 0x29, 0xac, 0x80, 0x19, 0x53, + 0xfe, 0xa5, 0x20, 0x1c, 0x9a, 0x31, 0x16, 0x44, 0x97, 0xd9, 0x91, 0xff, 0x25, 0x84, 0x74, 0xe3, + 0x67, 0x1f, 0x3e, 0x97, 0xa7, 0x3e, 0x7e, 0x2e, 0x4f, 0x7d, 0xfa, 0x5c, 0x9e, 0xfa, 0x6d, 0xbf, + 0xac, 0x7d, 0xe8, 0x97, 0xb5, 0x8f, 0xfd, 0xb2, 0xf6, 0xa9, 0x5f, 0xd6, 0xfe, 0xd7, 0x2f, 0x6b, + 0x7f, 0xf8, 0x7f, 0x79, 0xea, 0x6d, 0x69, 0xfc, 0xbf, 0xb5, 0xdf, 0x06, 0x00, 0x00, 0xff, 0xff, + 0xee, 0x44, 0x0b, 0xed, 0xe3, 0x15, 0x00, 0x00, } func (m *CSIDriver) Marshal() (dAtA []byte, err error) { @@ -826,6 +827,16 @@ func (m *CSIDriverSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SELinuxMount != nil { + i-- + if *m.SELinuxMount { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x40 + } if m.RequiresRepublish != nil { i-- if *m.RequiresRepublish { @@ -1795,6 +1806,9 @@ func (m *CSIDriverSpec) Size() (n int) { if m.RequiresRepublish != nil { n += 2 } + if m.SELinuxMount != nil { + n += 2 + } return n } @@ -2148,6 +2162,7 @@ func (this *CSIDriverSpec) String() string { `FSGroupPolicy:` + valueToStringGenerated(this.FSGroupPolicy) + `,`, `TokenRequests:` + repeatedStringForTokenRequests + `,`, `RequiresRepublish:` + valueToStringGenerated(this.RequiresRepublish) + `,`, + `SELinuxMount:` + valueToStringGenerated(this.SELinuxMount) + `,`, `}`, }, "") return s @@ -2844,6 +2859,27 @@ func (m *CSIDriverSpec) Unmarshal(dAtA []byte) error { } b := bool(v != 0) m.RequiresRepublish = &b + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SELinuxMount", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + b := bool(v != 0) + m.SELinuxMount = &b default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/storage/v1beta1/generated.proto b/staging/src/k8s.io/api/storage/v1beta1/generated.proto index 943900fa690..bedbd318389 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/storage/v1beta1/generated.proto @@ -192,6 +192,27 @@ message CSIDriverSpec { // // +optional optional bool requiresRepublish = 7; + + // SELinuxMount specifies if the CSI driver supports "-o context" + // mount option. + // + // When "true", the CSI driver must ensure that all volumes provided by this CSI + // driver can be mounted separately with different `-o context` options. This is + // typical for storage backends that provide volumes as filesystems on block + // devices or as independent shared volumes. + // Kubernetes will call NodeStage / NodePublish with "-o context=xyz" mount + // option when mounting a ReadWriteOncePod volume used in Pod that has + // explicitly set SELinux context. In the future, it may be expanded to other + // volume AccessModes. In any case, Kubernetes will ensure that the volume is + // mounted only with a single SELinux context. + // + // When "false", Kubernetes won't pass any special SELinux mount options to the driver. + // This is typical for volumes that represent subdirectories of a bigger shared filesystem. + // + // Default is "false". + // + // +optional + optional bool seLinuxMount = 8; } // DEPRECATED - This group version of CSINode is deprecated by storage/v1/CSINode. diff --git a/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go index d810b4e4c8b..ea3c1e4c282 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go @@ -56,6 +56,7 @@ var map_CSIDriverSpec = map[string]string{ "fsGroupPolicy": "Defines if the underlying volume supports changing ownership and permission of the volume before being mounted. Refer to the specific FSGroupPolicy values for additional details.\n\nThis field is immutable.\n\nDefaults to ReadWriteOnceWithFSType, which will examine each volume to determine if Kubernetes should modify ownership and permissions of the volume. With the default policy the defined fsGroup will only be applied if a fstype is defined and the volume's access mode contains ReadWriteOnce.", "tokenRequests": "TokenRequests indicates the CSI driver needs pods' service account tokens it is mounting volume for to do necessary authentication. Kubelet will pass the tokens in VolumeContext in the CSI NodePublishVolume calls. The CSI driver should parse and validate the following VolumeContext: \"csi.storage.k8s.io/serviceAccount.tokens\": {\n \"\": {\n \"token\": ,\n \"expirationTimestamp\": ,\n },\n ...\n}\n\nNote: Audience in each TokenRequest should be different and at most one token is empty string. To receive a new token after expiry, RequiresRepublish can be used to trigger NodePublishVolume periodically.", "requiresRepublish": "RequiresRepublish indicates the CSI driver wants `NodePublishVolume` being periodically called to reflect any possible change in the mounted volume. This field defaults to false.\n\nNote: After a successful initial NodePublishVolume call, subsequent calls to NodePublishVolume should only update the contents of the volume. New mount points will not be seen by a running container.", + "seLinuxMount": "SELinuxMount specifies if the CSI driver supports \"-o context\" mount option.\n\nWhen \"true\", the CSI driver must ensure that all volumes provided by this CSI driver can be mounted separately with different `-o context` options. This is typical for storage backends that provide volumes as filesystems on block devices or as independent shared volumes. Kubernetes will call NodeStage / NodePublish with \"-o context=xyz\" mount option when mounting a ReadWriteOncePod volume used in Pod that has explicitly set SELinux context. In the future, it may be expanded to other volume AccessModes. In any case, Kubernetes will ensure that the volume is mounted only with a single SELinux context.\n\nWhen \"false\", Kubernetes won't pass any special SELinux mount options to the driver. This is typical for volumes that represent subdirectories of a bigger shared filesystem.\n\nDefault is \"false\".", } func (CSIDriverSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go index 5411ed8c00e..f0450182b27 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go @@ -127,6 +127,11 @@ func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = new(bool) **out = **in } + if in.SELinuxMount != nil { + in, out := &in.SELinuxMount, &out.SELinuxMount + *out = new(bool) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.json b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.json index a8fd91656a1..3acd8ccac4d 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.json +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.json @@ -57,6 +57,7 @@ "expirationSeconds": 2 } ], - "requiresRepublish": true + "requiresRepublish": true, + "seLinuxMount": true } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.pb b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.pb index 90474178b5d08c11e9290b1518e96d560c5c7971..6b6621c28b86ba2ee83fcb255c60d2f8b74ad861 100644 GIT binary patch delta 25 gcmcb`e2000D&xkDYN?EjK9hSGz1SQWr5KbL0Bs%yn*aa+ delta 23 ecmcb^e2aO4D&zW%YN?EjUXyzmy;!6elo$YCBL-9e diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.yaml b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.yaml index c2d81056533..b787db002e0 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1.CSIDriver.yaml @@ -37,6 +37,7 @@ spec: fsGroupPolicy: fsGroupPolicyValue podInfoOnMount: true requiresRepublish: true + seLinuxMount: true storageCapacity: true tokenRequests: - audience: audienceValue diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json index 4c810052d9f..f3d7b2be855 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json @@ -57,6 +57,7 @@ "expirationSeconds": 2 } ], - "requiresRepublish": true + "requiresRepublish": true, + "seLinuxMount": true } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.pb b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.pb index 4516309f96d7f3b509f61564f406eb15930b2a81..17b4b4a0a4635812010c3b1383d5a6274f5f47f0 100644 GIT binary patch delta 25 hcmcc5{E&Hq7URZ^+Nq3;K9eUhda*e$N--!g003_E2DAVG delta 23 fcmaFJe4lxO7UTMj+Nq3;UXv#>da+0`C@}y4V8;e+ diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml index 07adc3365d9..5fd85ceaaeb 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml @@ -37,6 +37,7 @@ spec: fsGroupPolicy: fsGroupPolicyValue podInfoOnMount: true requiresRepublish: true + seLinuxMount: true storageCapacity: true tokenRequests: - audience: audienceValue diff --git a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go index 5b51d38f1b0..e3e6183c097 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go @@ -10843,6 +10843,9 @@ var schemaYAML = typed.YAMLObject(`types: - name: requiresRepublish type: scalar: boolean + - name: seLinuxMount + type: + scalar: boolean - name: storageCapacity type: scalar: boolean @@ -11195,6 +11198,9 @@ var schemaYAML = typed.YAMLObject(`types: - name: requiresRepublish type: scalar: boolean + - name: seLinuxMount + type: + scalar: boolean - name: storageCapacity type: scalar: boolean diff --git a/staging/src/k8s.io/client-go/applyconfigurations/storage/v1/csidriverspec.go b/staging/src/k8s.io/client-go/applyconfigurations/storage/v1/csidriverspec.go index 1dc17ce96a8..a1ef00656bb 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/storage/v1/csidriverspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/storage/v1/csidriverspec.go @@ -32,6 +32,7 @@ type CSIDriverSpecApplyConfiguration struct { FSGroupPolicy *v1.FSGroupPolicy `json:"fsGroupPolicy,omitempty"` TokenRequests []TokenRequestApplyConfiguration `json:"tokenRequests,omitempty"` RequiresRepublish *bool `json:"requiresRepublish,omitempty"` + SELinuxMount *bool `json:"seLinuxMount,omitempty"` } // CSIDriverSpecApplyConfiguration constructs an declarative configuration of the CSIDriverSpec type for use with @@ -102,3 +103,11 @@ func (b *CSIDriverSpecApplyConfiguration) WithRequiresRepublish(value bool) *CSI b.RequiresRepublish = &value return b } + +// WithSELinuxMount sets the SELinuxMount field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SELinuxMount field is set to the value of the last call. +func (b *CSIDriverSpecApplyConfiguration) WithSELinuxMount(value bool) *CSIDriverSpecApplyConfiguration { + b.SELinuxMount = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/storage/v1beta1/csidriverspec.go b/staging/src/k8s.io/client-go/applyconfigurations/storage/v1beta1/csidriverspec.go index 1d943cbfffe..6097a615be2 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/storage/v1beta1/csidriverspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/storage/v1beta1/csidriverspec.go @@ -32,6 +32,7 @@ type CSIDriverSpecApplyConfiguration struct { FSGroupPolicy *v1beta1.FSGroupPolicy `json:"fsGroupPolicy,omitempty"` TokenRequests []TokenRequestApplyConfiguration `json:"tokenRequests,omitempty"` RequiresRepublish *bool `json:"requiresRepublish,omitempty"` + SELinuxMount *bool `json:"seLinuxMount,omitempty"` } // CSIDriverSpecApplyConfiguration constructs an declarative configuration of the CSIDriverSpec type for use with @@ -102,3 +103,11 @@ func (b *CSIDriverSpecApplyConfiguration) WithRequiresRepublish(value bool) *CSI b.RequiresRepublish = &value return b } + +// WithSELinuxMount sets the SELinuxMount field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SELinuxMount field is set to the value of the last call. +func (b *CSIDriverSpecApplyConfiguration) WithSELinuxMount(value bool) *CSIDriverSpecApplyConfiguration { + b.SELinuxMount = &value + return b +} From f99cf5180e020c263d7b0e6954b4bd26bb89a054 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 11 Feb 2022 10:48:35 +0100 Subject: [PATCH 05/22] Add SELinux mount option to NewMounter() and MountDevice() Let volume plugins decide if they want to mount volumes with "-o context=XYZ" or let the container runtime relabel the volume on container startup. Using NewMounter, as it's the call where a volume plugin gets the other MountOptions. --- pkg/volume/volume.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index f46f7d4876a..75c2d906151 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -129,6 +129,7 @@ type MounterArgs struct { FsGroup *int64 FSGroupChangePolicy *v1.PodFSGroupChangePolicy DesiredSize *resource.Quantity + SELinuxLabel string } // Mounter interface provides methods to set up/mount the volume. @@ -262,7 +263,8 @@ type Attacher interface { // DeviceMounterArgs provides auxiliary, optional arguments to DeviceMounter. type DeviceMounterArgs struct { - FsGroup *int64 + FsGroup *int64 + SELinuxLabel string } // DeviceMounter can mount a block volume to a global path. From cdb3ead5a9bbf292b3efeca88781b00d7d061082 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 11 Feb 2022 12:30:19 +0100 Subject: [PATCH 06/22] Add SupportsSELinuxContextMount Add a new call to VolumePlugin interface and change all its implementations. Kubelet's VolumeManager will be interested whether a volume supports mounting with -o conext=XYZ or not to hanle SetUp() / MountDevice() accordingly. --- .../volume/attachdetach/testing/testvolumespec.go | 4 ++++ .../volume/persistentvolume/framework_test.go | 4 ++++ pkg/volume/awsebs/aws_ebs.go | 4 ++++ pkg/volume/azure_file/azure_file.go | 4 ++++ pkg/volume/azuredd/azure_dd.go | 4 ++++ pkg/volume/cephfs/cephfs.go | 4 ++++ pkg/volume/cinder/cinder.go | 4 ++++ pkg/volume/configmap/configmap.go | 4 ++++ pkg/volume/csi/csi_plugin.go | 4 ++++ pkg/volume/downwardapi/downwardapi.go | 4 ++++ pkg/volume/emptydir/empty_dir.go | 4 ++++ pkg/volume/fc/fc.go | 4 ++++ pkg/volume/flexvolume/plugin.go | 4 ++++ pkg/volume/gcepd/gce_pd.go | 4 ++++ pkg/volume/git_repo/git_repo.go | 4 ++++ pkg/volume/glusterfs/glusterfs.go | 4 ++++ pkg/volume/hostpath/host_path.go | 4 ++++ pkg/volume/iscsi/iscsi.go | 4 ++++ pkg/volume/local/local.go | 4 ++++ pkg/volume/nfs/nfs.go | 4 ++++ pkg/volume/noop_expandable_plugin.go | 4 ++++ pkg/volume/plugins.go | 4 ++++ pkg/volume/plugins_test.go | 4 ++++ pkg/volume/portworx/portworx.go | 4 ++++ pkg/volume/projected/projected.go | 4 ++++ pkg/volume/rbd/rbd.go | 4 ++++ pkg/volume/secret/secret.go | 4 ++++ pkg/volume/testing/testing.go | 12 ++++++++++++ pkg/volume/vsphere_volume/vsphere_volume.go | 4 ++++ 29 files changed, 124 insertions(+) diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index c9586b75dd7..0af3f0b52d4 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -483,6 +483,10 @@ func (plugin *TestPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *TestPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *TestPlugin) GetErrorEncountered() bool { plugin.pluginLock.RLock() defer plugin.pluginLock.RUnlock() diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index ebf98ab8558..6ec4f86781c 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -970,6 +970,10 @@ func (plugin *mockVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string return nil, nil } +func (plugin *mockVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *mockVolumePlugin) NewMounter(spec *volume.Spec, podRef *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return nil, fmt.Errorf("Mounter is not supported by this plugin") } diff --git a/pkg/volume/awsebs/aws_ebs.go b/pkg/volume/awsebs/aws_ebs.go index 6d4752d17a7..c50e8e03635 100644 --- a/pkg/volume/awsebs/aws_ebs.go +++ b/pkg/volume/awsebs/aws_ebs.go @@ -100,6 +100,10 @@ func (plugin *awsElasticBlockStorePlugin) SupportsBulkVolumeVerification() bool return true } +func (plugin *awsElasticBlockStorePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *awsElasticBlockStorePlugin) GetVolumeLimits() (map[string]int64, error) { volumeLimits := map[string]int64{ util.EBSVolumeLimitKey: util.DefaultMaxEBSVolumes, diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index 61953cf3d48..df331ea9f48 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -100,6 +100,10 @@ func (plugin *azureFilePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *azureFilePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *azureFilePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/azuredd/azure_dd.go b/pkg/volume/azuredd/azure_dd.go index e839474483c..25dd97af883 100644 --- a/pkg/volume/azuredd/azure_dd.go +++ b/pkg/volume/azuredd/azure_dd.go @@ -134,6 +134,10 @@ func (plugin *azureDataDiskPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *azureDataDiskPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *azureDataDiskPlugin) GetVolumeLimits() (map[string]int64, error) { volumeLimits := map[string]int64{ util.AzureVolumeLimitKey: defaultAzureVolumeLimit, diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go index 2a6e8800514..2fc27d6248b 100644 --- a/pkg/volume/cephfs/cephfs.go +++ b/pkg/volume/cephfs/cephfs.go @@ -85,6 +85,10 @@ func (plugin *cephfsPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *cephfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *cephfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 88f89489aec..ef422e24c5c 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -123,6 +123,10 @@ func (plugin *cinderPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *cinderPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + var _ volume.VolumePluginWithAttachLimits = &cinderPlugin{} func (plugin *cinderPlugin) GetVolumeLimits() (map[string]int64, error) { diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go index e72f5bccb70..8aca9bc2314 100644 --- a/pkg/volume/configmap/configmap.go +++ b/pkg/volume/configmap/configmap.go @@ -90,6 +90,10 @@ func (plugin *configMapPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *configMapPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *configMapPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return &configMapVolumeMounter{ configMapVolume: &configMapVolume{ diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index f922fec485a..25358644e85 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -581,6 +581,10 @@ func (p *csiPlugin) SupportsBulkVolumeVerification() bool { return false } +func (p *csiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + // volume.AttachableVolumePlugin methods var _ volume.AttachableVolumePlugin = &csiPlugin{} diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go index 08899bfb359..714254c5c79 100644 --- a/pkg/volume/downwardapi/downwardapi.go +++ b/pkg/volume/downwardapi/downwardapi.go @@ -92,6 +92,10 @@ func (plugin *downwardAPIPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *downwardAPIPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *downwardAPIPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { v := &downwardAPIVolume{ volName: spec.Name(), diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index 19cd6aea282..6370902d0ea 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -103,6 +103,10 @@ func (plugin *emptyDirPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *emptyDirPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *emptyDirPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()), &realMountDetector{plugin.host.GetMounter(plugin.GetPluginName())}, opts) } diff --git a/pkg/volume/fc/fc.go b/pkg/volume/fc/fc.go index 3bb73f943a7..8c8920d6dc2 100644 --- a/pkg/volume/fc/fc.go +++ b/pkg/volume/fc/fc.go @@ -99,6 +99,10 @@ func (plugin *fcPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *fcPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *fcPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index a13b3af9326..92e5c8d91db 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -287,6 +287,10 @@ func (plugin *flexVolumePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *flexVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + // Returns true iff the given command is known to be unsupported. func (plugin *flexVolumePlugin) isUnsupported(command string) bool { plugin.Lock() diff --git a/pkg/volume/gcepd/gce_pd.go b/pkg/volume/gcepd/gce_pd.go index 302d6be9613..cb33c1e3dd3 100644 --- a/pkg/volume/gcepd/gce_pd.go +++ b/pkg/volume/gcepd/gce_pd.go @@ -111,6 +111,10 @@ func (plugin *gcePersistentDiskPlugin) SupportsBulkVolumeVerification() bool { return true } +func (plugin *gcePersistentDiskPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *gcePersistentDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/git_repo/git_repo.go b/pkg/volume/git_repo/git_repo.go index 5f0e7075eb7..76dafd7c839 100644 --- a/pkg/volume/git_repo/git_repo.go +++ b/pkg/volume/git_repo/git_repo.go @@ -89,6 +89,10 @@ func (plugin *gitRepoPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *gitRepoPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *gitRepoPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { if err := validateVolume(spec.Volume.GitRepo); err != nil { return nil, err diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index 07bdc926719..10e571a8b5d 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -132,6 +132,10 @@ func (plugin *glusterfsPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *glusterfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *glusterfsPlugin) RequiresFSResize() bool { return false } diff --git a/pkg/volume/hostpath/host_path.go b/pkg/volume/hostpath/host_path.go index 8900023c51f..c6f5b4c779c 100644 --- a/pkg/volume/hostpath/host_path.go +++ b/pkg/volume/hostpath/host_path.go @@ -108,6 +108,10 @@ func (plugin *hostPathPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *hostPathPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *hostPathPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 830bd5f717b..f19f670c2c7 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -92,6 +92,10 @@ func (plugin *iscsiPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *iscsiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *iscsiPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index df9b7f560f9..ab2af54c2e4 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -96,6 +96,10 @@ func (plugin *localVolumePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *localVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *localVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { // The current meaning of AccessMode is how many nodes can attach to it, not how many pods can mount it return []v1.PersistentVolumeAccessMode{ diff --git a/pkg/volume/nfs/nfs.go b/pkg/volume/nfs/nfs.go index 4a557e884bd..f292be4a506 100644 --- a/pkg/volume/nfs/nfs.go +++ b/pkg/volume/nfs/nfs.go @@ -105,6 +105,10 @@ func (plugin *nfsPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *nfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *nfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/noop_expandable_plugin.go b/pkg/volume/noop_expandable_plugin.go index fac27426301..8e3872e3712 100644 --- a/pkg/volume/noop_expandable_plugin.go +++ b/pkg/volume/noop_expandable_plugin.go @@ -75,3 +75,7 @@ func (n *noopExpandableVolumePluginInstance) SupportsBulkVolumeVerification() bo func (n *noopExpandableVolumePluginInstance) RequiresFSResize() bool { return true } + +func (n *noopExpandableVolumePluginInstance) SupportsSELinuxContextMount(spec *Spec) (bool, error) { + return false, nil +} diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 5fd44568a01..23afc41539b 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -187,6 +187,10 @@ type VolumePlugin interface { // of enabling bulk polling of all nodes. This can speed up verification of // attached volumes by quite a bit, but underlying pluging must support it. SupportsBulkVolumeVerification() bool + + // SupportsSELinuxContextMount returns true if volume plugins supports + // mount -o context=XYZ for a given volume. + SupportsSELinuxContextMount(spec *Spec) (bool, error) } // PersistentVolumePlugin is an extended interface of VolumePlugin and is used diff --git a/pkg/volume/plugins_test.go b/pkg/volume/plugins_test.go index 510f204ca1b..dabb421a344 100644 --- a/pkg/volume/plugins_test.go +++ b/pkg/volume/plugins_test.go @@ -87,6 +87,10 @@ func (plugin *testPlugins) SupportsBulkVolumeVerification() bool { return false } +func (plugin *testPlugins) SupportsSELinuxContextMount(spec *Spec) (bool, error) { + return false, nil +} + func (plugin *testPlugins) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) { return nil, nil } diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index b5429b9de61..417929b780d 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -230,6 +230,10 @@ func (plugin *portworxVolumePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *portworxVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func getVolumeSource( spec *volume.Spec) (*v1.PortworxVolumeSource, bool, error) { if spec.Volume != nil && spec.Volume.PortworxVolume != nil { diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go index e30cb802243..ecbe408098e 100644 --- a/pkg/volume/projected/projected.go +++ b/pkg/volume/projected/projected.go @@ -105,6 +105,10 @@ func (plugin *projectedPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *projectedPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *projectedPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return &projectedVolumeMounter{ projectedVolume: &projectedVolume{ diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 85f949a5b55..adb4e61376c 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -125,6 +125,10 @@ func (plugin *rbdPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *rbdPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index 0899f512667..a8a2d633b5c 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -93,6 +93,10 @@ func (plugin *secretPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *secretPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *secretPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return &secretVolumeMounter{ secretVolume: &secretVolume{ diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index a747d755725..65949ad148e 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -284,6 +284,10 @@ func (plugin *FakeVolumePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *FakeVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *FakeVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { plugin.Lock() defer plugin.Unlock() @@ -545,6 +549,10 @@ func (f *FakeBasicVolumePlugin) SupportsBulkVolumeVerification() bool { return f.Plugin.SupportsBulkVolumeVerification() } +func (f *FakeBasicVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return f.Plugin.SupportsSELinuxContextMount(spec) +} + func (f *FakeBasicVolumePlugin) SupportsMountOption() bool { return f.Plugin.SupportsMountOption() } @@ -626,6 +634,10 @@ func (plugin *FakeFileVolumePlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *FakeFileVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *FakeFileVolumePlugin) NewMounter(spec *volume.Spec, podRef *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { return nil, nil } diff --git a/pkg/volume/vsphere_volume/vsphere_volume.go b/pkg/volume/vsphere_volume/vsphere_volume.go index 904f3688669..15072690373 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume.go +++ b/pkg/volume/vsphere_volume/vsphere_volume.go @@ -104,6 +104,10 @@ func (plugin *vsphereVolumePlugin) SupportsBulkVolumeVerification() bool { return true } +func (plugin *vsphereVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + return false, nil +} + func (plugin *vsphereVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { return plugin.newMounterInternal(spec, pod.UID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } From 4cfb277e8b896af52d038157a829b6577e8379f8 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 11 Feb 2022 10:50:03 +0100 Subject: [PATCH 07/22] Implement mounting with -o context= in iSCSI volume plugin --- pkg/volume/iscsi/attacher.go | 5 ++++- pkg/volume/iscsi/iscsi.go | 22 +++++++++++++--------- pkg/volume/util/util.go | 12 ++++++++++++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pkg/volume/iscsi/attacher.go b/pkg/volume/iscsi/attacher.go index f4871d901af..8aa184c3a59 100644 --- a/pkg/volume/iscsi/attacher.go +++ b/pkg/volume/iscsi/attacher.go @@ -98,7 +98,7 @@ func (attacher *iscsiAttacher) GetDeviceMountPath( return attacher.manager.MakeGlobalPDName(*mounter.iscsiDisk), nil } -func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error { +func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mountArgs volume.DeviceMounterArgs) error { mounter := attacher.host.GetMounter(iscsiPluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { @@ -120,6 +120,9 @@ func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, if readOnly { options = append(options, "ro") } + if mountArgs.SELinuxLabel != "" { + options = volumeutil.AddSELinuxMountOption(options, mountArgs.SELinuxLabel) + } if notMnt { diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(iscsiPluginName)} mountOptions := volumeutil.MountOptionFromSpec(spec, options...) diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index f19f670c2c7..7390e59b341 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -93,7 +93,7 @@ func (plugin *iscsiPlugin) SupportsBulkVolumeVerification() bool { } func (plugin *iscsiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { - return false, nil + return true, nil } func (plugin *iscsiPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { @@ -336,13 +336,14 @@ func (iscsi *iscsiDisk) iscsiPodDeviceMapPath() (string, string) { type iscsiDiskMounter struct { *iscsiDisk - readOnly bool - fsType string - volumeMode v1.PersistentVolumeMode - mounter *mount.SafeFormatAndMount - exec utilexec.Interface - deviceUtil ioutil.DeviceUtil - mountOptions []string + readOnly bool + fsType string + volumeMode v1.PersistentVolumeMode + mounter *mount.SafeFormatAndMount + exec utilexec.Interface + deviceUtil ioutil.DeviceUtil + mountOptions []string + mountedWithSELinuxContext bool } var _ volume.Mounter = &iscsiDiskMounter{} @@ -351,7 +352,7 @@ func (b *iscsiDiskMounter) GetAttributes() volume.Attributes { return volume.Attributes{ ReadOnly: b.readOnly, Managed: !b.readOnly, - SELinuxRelabel: true, + SELinuxRelabel: !b.mountedWithSELinuxContext, } } @@ -365,6 +366,9 @@ func (b *iscsiDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) e if err != nil { klog.Errorf("iscsi: failed to setup") } + // The volume must have been mounted in MountDevice with -o context. + // TODO: extract from mount table in GetAttributes() to be sure? + b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != "" return err } diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index 807ea4b379f..dae2ec59c37 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -35,11 +35,13 @@ import ( utypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" storagehelpers "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/securitycontext" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/types" @@ -273,6 +275,16 @@ func JoinMountOptions(userOptions []string, systemOptions []string) []string { return allMountOptions.List() } +// AddSELinuxMountOption adds -o context="XYZ mount option to a given list +func AddSELinuxMountOption(options []string, seLinuxContext string) []string { + if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + return options + } + // Use double quotes to support a comma "," in the SELinux context string. + // For example: dirsync,context="system_u:object_r:container_file_t:s0:c15,c25",noatime + return append(options, "context=%q", seLinuxContext) +} + // ContainsAccessMode returns whether the requested mode is contained by modes func ContainsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool { for _, m := range modes { From 48b0751269e60129bb8b8cdbf2dd91c7a918687f Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 11 Feb 2022 10:41:16 +0100 Subject: [PATCH 08/22] Add SELinux context tracking to volume manager Both ActualStateOfWorld and DesiredStateOfWorld must track SELinux context of volume mounts. --- .../cache/actual_state_of_world.go | 121 ++++++++++++++--- .../cache/actual_state_of_world_test.go | 8 +- .../cache/desired_state_of_world.go | 123 ++++++++++++++++-- .../cache/desired_state_of_world_test.go | 25 ++-- .../volumemanager/metrics/metrics_test.go | 4 +- .../desired_state_of_world_populator.go | 4 +- .../volumemanager/reconciler/reconciler.go | 16 ++- .../reconciler/reconciler_test.go | 32 ++--- pkg/kubelet/volumemanager/volume_manager.go | 2 +- .../operationexecutor/operation_executor.go | 16 ++- .../operationexecutor/operation_generator.go | 18 +-- pkg/volume/util/selinux.go | 117 +++++++++++++++++ pkg/volume/util/util.go | 28 ++-- 13 files changed, 421 insertions(+), 93 deletions(-) create mode 100644 pkg/volume/util/selinux.go diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 40c7d943c7e..f20ae8be04b 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -27,7 +27,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -73,7 +75,7 @@ type ActualStateOfWorld interface { // global mount point prior to detach. // If a volume with the name volumeName does not exist in the list of // attached volumes, an error is returned. - SetDeviceMountState(volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath string) error + SetDeviceMountState(volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath, seLinuxMountContext string) error // DeletePodFromVolume removes the given pod from the given volume in the // cache indicating the volume has been successfully unmounted from the pod. @@ -107,7 +109,7 @@ type ActualStateOfWorld interface { // volumes, depend on this to update the contents of the volume. // All volume mounting calls should be idempotent so a second mount call for // volumes that do not need to update contents should not fail. - PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity) (bool, string, error) + PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity, seLinuxLabel string) (bool, string, error) // PodRemovedFromVolume returns true if the given pod does not exist in the list of // mountedPods for the given volume in the cache, indicating that the pod has @@ -182,6 +184,11 @@ type AttachedVolume struct { // DeviceMountState indicates if device has been globally mounted or is not. DeviceMountState operationexecutor.DeviceMountState + + // SELinuxMountContext is the context with that the volume is globally mounted + // (via -o context=XYZ mount option). If empty, the volume is not mounted with + // "-o context=". + SELinuxMountContext string } // DeviceMayBeMounted returns true if device is mounted in global path or is in @@ -288,6 +295,11 @@ type attachedVolume struct { // persistentVolumeSize records size of the volume when pod was started or // size after successful completion of volume expansion operation. persistentVolumeSize *resource.Quantity + + // seLinuxMountContext is the context with that the volume is mounted to global directory + // (via -o context=XYZ mount option). If nil, the volume is not mounted. If "", the volume is + // mounted without "-o context=". + seLinuxMountContext *string } // The mountedPod object represents a pod for which the kubelet volume manager @@ -333,6 +345,11 @@ type mountedPod struct { // - VolumeMounted: means volume for pod has been successfully mounted // - VolumeMountUncertain: means volume for pod may not be mounted, but it must be unmounted volumeMountStateForPod operationexecutor.VolumeMountState + + // seLinuxMountContext is the context with that the volume is mounted to Pod directory + // (via -o context=XYZ mount option). If nil, the volume is not mounted. If "", the volume is + // mounted without "-o context=". + seLinuxMountContext string } func (asw *actualStateOfWorld) MarkVolumeAsAttached( @@ -465,13 +482,13 @@ func (asw *actualStateOfWorld) MarkVolumeAsUnmounted( } func (asw *actualStateOfWorld) MarkDeviceAsMounted( - volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error { - return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceGloballyMounted, devicePath, deviceMountPath) + volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error { + return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceGloballyMounted, devicePath, deviceMountPath, seLinuxMountContext) } func (asw *actualStateOfWorld) MarkDeviceAsUncertain( - volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error { - return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceMountUncertain, devicePath, deviceMountPath) + volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error { + return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceMountUncertain, devicePath, deviceMountPath, seLinuxMountContext) } func (asw *actualStateOfWorld) MarkVolumeMountAsUncertain(markVolumeOpts operationexecutor.MarkVolumeOpts) error { @@ -481,7 +498,7 @@ func (asw *actualStateOfWorld) MarkVolumeMountAsUncertain(markVolumeOpts operati func (asw *actualStateOfWorld) MarkDeviceAsUnmounted( volumeName v1.UniqueVolumeName) error { - return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceNotMounted, "", "") + return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceNotMounted, "", "", "") } func (asw *actualStateOfWorld) GetDeviceMountState(volumeName v1.UniqueVolumeName) operationexecutor.DeviceMountState { @@ -629,6 +646,7 @@ func (asw *actualStateOfWorld) AddPodToVolume(markVolumeOpts operationexecutor.M volumeGidValue: volumeGidValue, volumeSpec: volumeSpec, volumeMountStateForPod: markVolumeOpts.VolumeMountState, + seLinuxMountContext: markVolumeOpts.SELinuxMountContext, } } @@ -646,6 +664,15 @@ func (asw *actualStateOfWorld) AddPodToVolume(markVolumeOpts operationexecutor.M podObj.mounter = mounter } asw.attachedVolumes[volumeName].mountedPods[podName] = podObj + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + // Store the mount context also in the AttachedVolume to have a global volume context + // for a quick comparison in PodExistsInVolume. + if volumeObj.seLinuxMountContext == nil { + volumeObj.seLinuxMountContext = &markVolumeOpts.SELinuxMountContext + asw.attachedVolumes[volumeName] = volumeObj + } + } + return nil } @@ -685,7 +712,7 @@ func (asw *actualStateOfWorld) MarkRemountRequired( } func (asw *actualStateOfWorld) SetDeviceMountState( - volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath string) error { + volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath, seLinuxMountContext string) error { asw.Lock() defer asw.Unlock() @@ -701,6 +728,11 @@ func (asw *actualStateOfWorld) SetDeviceMountState( if devicePath != "" { volumeObj.devicePath = devicePath } + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + if seLinuxMountContext != "" { + volumeObj.seLinuxMountContext = &seLinuxMountContext + } + } asw.attachedVolumes[volumeName] = volumeObj return nil } @@ -776,7 +808,7 @@ func (asw *actualStateOfWorld) DeleteVolume(volumeName v1.UniqueVolumeName) erro return nil } -func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity) (bool, string, error) { +func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity, seLinuxLabel string) (bool, string, error) { asw.RLock() defer asw.RUnlock() @@ -785,6 +817,22 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa return false, "", newVolumeNotAttachedError(volumeName) } + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + if volumeObj.seLinuxMountContext != nil { + // The volume is mounted, check its SELinux context mount option + if *volumeObj.seLinuxMountContext != seLinuxLabel { + fullErr := newSELinuxMountMismatchError(volumeName) + if util.IsRWOP(volumeObj.spec) { + return false, volumeObj.devicePath, fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + // TODO: bump some metric here + klog.V(4).ErrorS(fullErr, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + } + } + } + } + podObj, podExists := volumeObj.mountedPods[podName] if podExists { // if volume mount was uncertain we should keep trying to mount the volume @@ -905,7 +953,6 @@ func (asw *actualStateOfWorld) GetAllMountedVolumes() []MountedVolume { mountedVolume, getMountedVolume(&podObj, &volumeObj)) } - } } @@ -1010,15 +1057,22 @@ func (asw *actualStateOfWorld) SyncReconstructedVolume(volumeName v1.UniqueVolum func (asw *actualStateOfWorld) newAttachedVolume( attachedVolume *attachedVolume) AttachedVolume { + seLinuxMountContext := "" + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + if attachedVolume.seLinuxMountContext != nil { + seLinuxMountContext = *attachedVolume.seLinuxMountContext + } + } return AttachedVolume{ AttachedVolume: operationexecutor.AttachedVolume{ - VolumeName: attachedVolume.volumeName, - VolumeSpec: attachedVolume.spec, - NodeName: asw.nodeName, - PluginIsAttachable: attachedVolume.pluginIsAttachable, - DevicePath: attachedVolume.devicePath, - DeviceMountPath: attachedVolume.deviceMountPath, - PluginName: attachedVolume.pluginName}, + VolumeName: attachedVolume.volumeName, + VolumeSpec: attachedVolume.spec, + NodeName: asw.nodeName, + PluginIsAttachable: attachedVolume.pluginIsAttachable, + DevicePath: attachedVolume.devicePath, + DeviceMountPath: attachedVolume.deviceMountPath, + PluginName: attachedVolume.pluginName, + SELinuxMountContext: seLinuxMountContext}, DeviceMountState: attachedVolume.deviceMountState, } } @@ -1105,6 +1159,10 @@ func IsFSResizeRequiredError(err error) bool { // mountedPod and attachedVolume objects. func getMountedVolume( mountedPod *mountedPod, attachedVolume *attachedVolume) MountedVolume { + seLinuxMountContext := "" + if attachedVolume.seLinuxMountContext != nil { + seLinuxMountContext = *attachedVolume.seLinuxMountContext + } return MountedVolume{ MountedVolume: operationexecutor.MountedVolume{ PodName: mountedPod.podName, @@ -1117,5 +1175,32 @@ func getMountedVolume( BlockVolumeMapper: mountedPod.blockVolumeMapper, VolumeGidValue: mountedPod.volumeGidValue, VolumeSpec: mountedPod.volumeSpec, - DeviceMountPath: attachedVolume.deviceMountPath}} + DeviceMountPath: attachedVolume.deviceMountPath, + SELinuxMountContext: seLinuxMountContext}} + +} + +// seLinuxMountMismatchError is an error returned when PodExistsInVolume() found +// a volume mounted with a different SELinux label than expected. +type seLinuxMountMismatchError struct { + volumeName v1.UniqueVolumeName +} + +func (err seLinuxMountMismatchError) Error() string { + return fmt.Sprintf( + "volumeName %q is already mounted to a different pod with a different SELinux label", + err.volumeName) +} + +func newSELinuxMountMismatchError(volumeName v1.UniqueVolumeName) error { + return seLinuxMountMismatchError{ + volumeName: volumeName, + } +} + +// IsSELinuxMountMismatchError returns true if the specified error is a +// seLinuxMountMismatchError. +func IsSELinuxMountMismatchError(err error) bool { + _, ok := err.(seLinuxMountMismatchError) + return ok } diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go index 019cfabf342..ecbb35fe55b 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go @@ -737,7 +737,7 @@ func Test_MarkDeviceAsMounted_Positive_NewVolume(t *testing.T) { } // Act - err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath, deviceMountPath) + err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath, deviceMountPath, "") // Assert if err != nil { @@ -824,7 +824,7 @@ func TestUncertainVolumeMounts(t *testing.T) { t.Fatalf("expected volume %s to be found in aws.GetPossiblyMountedVolumesForPod", volumeSpec1.Name()) } - volExists, _, _ := asw.PodExistsInVolume(podName1, generatedVolumeName1, resource.Quantity{}) + volExists, _, _ := asw.PodExistsInVolume(podName1, generatedVolumeName1, resource.Quantity{}, "") if volExists { t.Fatalf("expected volume %s to not exist in asw", generatedVolumeName1) } @@ -910,7 +910,7 @@ func verifyPodExistsInVolumeAsw( expectedDevicePath string, asw ActualStateOfWorld) { podExistsInVolume, devicePath, err := - asw.PodExistsInVolume(expectedPodName, expectedVolumeName, resource.Quantity{}) + asw.PodExistsInVolume(expectedPodName, expectedVolumeName, resource.Quantity{}, "") if err != nil { t.Fatalf( "ASW PodExistsInVolume failed. Expected: Actual: <%v>", err) @@ -952,7 +952,7 @@ func verifyPodDoesntExistInVolumeAsw( expectVolumeToExist bool, asw ActualStateOfWorld) { podExistsInVolume, devicePath, err := - asw.PodExistsInVolume(podToCheck, volumeToCheck, resource.Quantity{}) + asw.PodExistsInVolume(podToCheck, volumeToCheck, resource.Quantity{}, "") if !expectVolumeToExist && err == nil { t.Fatalf( "ASW PodExistsInVolume did not return error. Expected: Actual: <%v>", err) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 0a999833a5e..148f3775ca6 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -28,7 +28,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog/v2" apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -55,7 +58,7 @@ type DesiredStateOfWorld interface { // added. // If a pod with the same unique name already exists under the specified // volume, this is a no-op. - AddPodToVolume(podName types.UniquePodName, pod *v1.Pod, volumeSpec *volume.Spec, outerVolumeSpecName string, volumeGidValue string) (v1.UniqueVolumeName, error) + AddPodToVolume(podName types.UniquePodName, pod *v1.Pod, volumeSpec *volume.Spec, outerVolumeSpecName string, volumeGidValue string, seLinuxContainerContexts []*v1.SELinuxOptions) (v1.UniqueVolumeName, error) // MarkVolumesReportedInUse sets the ReportedInUse value to true for the // reportedVolumes. For volumes not in the reportedVolumes list, the @@ -83,7 +86,7 @@ type DesiredStateOfWorld interface { // volumes that should be attached to this node. // If a pod with the same unique name does not exist under the specified // volume, false is returned. - VolumeExists(volumeName v1.UniqueVolumeName) bool + VolumeExists(volumeName v1.UniqueVolumeName, seLinuxMountContext string) bool // PodExistsInVolume returns true if the given pod exists in the list of // podsToMount for the given volume in the cache. @@ -91,7 +94,7 @@ type DesiredStateOfWorld interface { // volume, false is returned. // If a volume with the name volumeName does not exist in the list of // attached volumes, false is returned. - PodExistsInVolume(podName types.UniquePodName, volumeName v1.UniqueVolumeName) bool + PodExistsInVolume(podName types.UniquePodName, volumeName v1.UniqueVolumeName, seLinuxMountContext string) bool // GetVolumesToMount generates and returns a list of volumes that should be // attached to this node and the pods they should be mounted to based on the @@ -195,6 +198,13 @@ type volumeToMount struct { // persistentVolumeSize records desired size of a persistent volume. // Usually this value reflects size recorded in pv.Spec.Capacity persistentVolumeSize *resource.Quantity + + // seLinuxFileLabel is desired SELinux label on files on the volume. If empty, then + // - either the context+label is unknown (assigned randomly by the container runtime) + // - or the volume plugin responsible for this volume does not support mounting with -o context + // - or the OS does not support SELinux + // In all cases, the SELinux context does not matter when mounting the volume. + seLinuxFileLabel string } // The pod object represents a pod that references the underlying volume and @@ -232,7 +242,8 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( pod *v1.Pod, volumeSpec *volume.Spec, outerVolumeSpecName string, - volumeGidValue string) (v1.UniqueVolumeName, error) { + volumeGidValue string, + seLinuxContainerContexts []*v1.SELinuxOptions) (v1.UniqueVolumeName, error) { dsw.Lock() defer dsw.Unlock() @@ -268,7 +279,63 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( volumeName = util.GetUniqueVolumeNameFromSpecWithPod(podName, volumePlugin, volumeSpec) } - if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { + var seLinuxFileLabel string + // Volume plugin supports SELinux context mount for all its volumes. + var pluginSupportsSELinuxContextMount bool + // The volume is ReadWriteOncePod. We don't support other volume types in SELinuxMountReadWriteOncePod feature. + // Don't use mount option to apply the SELinux context, still, track the context and report metrics of things + // that would break if the feature was for all volume access modes. + var isRWOP bool + + if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + pluginSupportsSELinuxContextMount, err = dsw.getSELinuxMountSupport(volumeSpec) + if err != nil { + return "", err + } + isRWOP = util.IsRWOP(volumeSpec) + if pluginSupportsSELinuxContextMount { + // Ensure that a volume that can be mounted with "-o context=XYZ" is + // used only by containers with the same SELinux contexts. + for _, containerContext := range seLinuxContainerContexts { + newLabel, err := util.SELinuxOptionsToFileLabel(containerContext) + if err != nil { + fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) + if isRWOP { + // Cannot mount with -o context if the context can't be composed. + return "", fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + // TODO: bump some metric here + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + break + } + } + if seLinuxFileLabel == "" { + seLinuxFileLabel = newLabel + continue + } + if seLinuxFileLabel != newLabel { + fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) + if isRWOP { + return "", fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + // TODO: bump some metric here + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + break + } + } + } + } else { + // Volume plugin does not support SELinux context mount. + // DSW will track this volume with SELinux label "", i.e. no mount with + // -o context. + seLinuxFileLabel = "" + } + } + klog.V(4).InfoS("volume final SELinux label decided", "volume", volumeSpec.Name(), "label", seLinuxFileLabel) + + if vol, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { var sizeLimit *resource.Quantity if volumeSpec.Volume != nil { if util.IsLocalEphemeralVolume(*volumeSpec.Volume) { @@ -291,6 +358,7 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( volumeGidValue: volumeGidValue, reportedInUse: false, desiredSizeLimit: sizeLimit, + seLinuxFileLabel: seLinuxFileLabel, } // record desired size of the volume if volumeSpec.PersistentVolume != nil { @@ -300,9 +368,24 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( vmt.persistentVolumeSize = &pvCapCopy } } - dsw.volumesToMount[volumeName] = vmt + } else { + // volume exists + if pluginSupportsSELinuxContextMount { + if seLinuxFileLabel != vol.seLinuxFileLabel { + // TODO: update the error message after tests, e.g. add at least the conflicting pod names. + fullErr := fmt.Errorf("conflicting SELinux labels of volume %s: %q and %q", volumeSpec.Name(), vol.seLinuxFileLabel, seLinuxFileLabel) + if isRWOP { + return "", fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + // TODO: bump some metric here + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + } + } + } } + oldPodMount, ok := dsw.volumesToMount[volumeName].podsToMount[podName] mountRequestTime := time.Now() if ok && !volumePlugin.RequiresRemount(volumeSpec) { @@ -380,16 +463,22 @@ func (dsw *desiredStateOfWorld) UpdatePersistentVolumeSize(volumeName v1.UniqueV } func (dsw *desiredStateOfWorld) VolumeExists( - volumeName v1.UniqueVolumeName) bool { + volumeName v1.UniqueVolumeName, seLinuxMountContext string) bool { dsw.RLock() defer dsw.RUnlock() - _, volumeExists := dsw.volumesToMount[volumeName] - return volumeExists + vol, volumeExists := dsw.volumesToMount[volumeName] + if !volumeExists { + return false + } + if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + return vol.seLinuxFileLabel == seLinuxMountContext + } + return true } func (dsw *desiredStateOfWorld) PodExistsInVolume( - podName types.UniquePodName, volumeName v1.UniqueVolumeName) bool { + podName types.UniquePodName, volumeName v1.UniqueVolumeName, seLinuxMountOption string) bool { dsw.RLock() defer dsw.RUnlock() @@ -398,6 +487,15 @@ func (dsw *desiredStateOfWorld) PodExistsInVolume( return false } + if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + if volumeObj.seLinuxFileLabel != seLinuxMountOption { + // The volume is in DSW, but with a different SELinux mount option. + // Report it as unused, so the volume is unmounted and mounted back + // with the right SELinux option. + return false + } + } + _, podExists := volumeObj.podsToMount[podName] return podExists } @@ -448,6 +546,7 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount { ReportedInUse: volumeObj.reportedInUse, MountRequestTime: podObj.mountRequestTime, DesiredSizeLimit: volumeObj.desiredSizeLimit, + SELinuxLabel: volumeObj.seLinuxFileLabel, }, } if volumeObj.persistentVolumeSize != nil { @@ -504,3 +603,7 @@ func (dsw *desiredStateOfWorld) MarkVolumeAttachability(volumeName v1.UniqueVolu volumeObj.pluginIsAttachable = attachable dsw.volumesToMount[volumeName] = volumeObj } + +func (dsw *desiredStateOfWorld) getSELinuxMountSupport(volumeSpec *volume.Spec) (bool, error) { + return util.SupportsSELinuxContextMount(volumeSpec, dsw.volumePluginMgr) +} diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go index 4777b32de1f..3d3d6639976 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go @@ -17,10 +17,11 @@ limitations under the License. package cache import ( - "k8s.io/apimachinery/pkg/api/resource" "testing" - "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/volume" volumetesting "k8s.io/kubernetes/pkg/volume/testing" @@ -59,7 +60,7 @@ func Test_AddPodToVolume_Positive_NewPodNewVolume(t *testing.T) { // Act generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) // Assert if err != nil { @@ -104,7 +105,7 @@ func Test_AddPodToVolume_Positive_ExistingPodExistingVolume(t *testing.T) { // Act generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) // Assert if err != nil { @@ -259,8 +260,8 @@ func Test_AddPodToVolume_Positive_NamesForDifferentPodsAndDifferentVolumes(t *te for name, v := range testcases { volumeSpec1 := &volume.Spec{Volume: &v.pod1.Spec.Volumes[0]} volumeSpec2 := &volume.Spec{Volume: &v.pod2.Spec.Volumes[0]} - generatedVolumeName1, err1 := dsw.AddPodToVolume(util.GetUniquePodName(v.pod1), v.pod1, volumeSpec1, volumeSpec1.Name(), "") - generatedVolumeName2, err2 := dsw.AddPodToVolume(util.GetUniquePodName(v.pod2), v.pod2, volumeSpec2, volumeSpec2.Name(), "") + generatedVolumeName1, err1 := dsw.AddPodToVolume(util.GetUniquePodName(v.pod1), v.pod1, volumeSpec1, volumeSpec1.Name(), "", nil) + generatedVolumeName2, err2 := dsw.AddPodToVolume(util.GetUniquePodName(v.pod2), v.pod2, volumeSpec2, volumeSpec2.Name(), "", nil) if err1 != nil { t.Fatalf("test %q: AddPodToVolume failed. Expected: Actual: <%v>", name, err1) } @@ -309,7 +310,7 @@ func Test_DeletePodFromVolume_Positive_PodExistsVolumeExists(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } @@ -407,19 +408,19 @@ func Test_MarkVolumesReportedInUse_Positive_NewPodNewVolume(t *testing.T) { pod3Name := util.GetUniquePodName(pod3) generatedVolume1Name, err := dsw.AddPodToVolume( - pod1Name, pod1, volume1Spec, volume1Spec.Name(), "" /* volumeGidValue */) + pod1Name, pod1, volume1Spec, volume1Spec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } generatedVolume2Name, err := dsw.AddPodToVolume( - pod2Name, pod2, volume2Spec, volume2Spec.Name(), "" /* volumeGidValue */) + pod2Name, pod2, volume2Spec, volume2Spec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } generatedVolume3Name, err := dsw.AddPodToVolume( - pod3Name, pod3, volume3Spec, volume3Spec.Name(), "" /* volumeGidValue */) + pod3Name, pod3, volume3Spec, volume3Spec.Name(), "" /* volumeGidValue */, nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } @@ -580,14 +581,14 @@ func Test_AddPodToVolume_WithEmptyDirSizeLimit(t *testing.T) { } for i := range pod1.Spec.Volumes { volumeSpec := &volume.Spec{Volume: &pod1.Spec.Volumes[i]} - _, err := dsw.AddPodToVolume(pod1Name, pod1, volumeSpec, volumeSpec.Name(), "") + _, err := dsw.AddPodToVolume(pod1Name, pod1, volumeSpec, volumeSpec.Name(), "", nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } } for i := range pod2.Spec.Volumes { volumeSpec := &volume.Spec{Volume: &pod2.Spec.Volumes[i]} - _, err := dsw.AddPodToVolume(pod2Name, pod2, volumeSpec, volumeSpec.Name(), "") + _, err := dsw.AddPodToVolume(pod2Name, pod2, volumeSpec, volumeSpec.Name(), "", nil /* seLinuxContainerContexts */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } diff --git a/pkg/kubelet/volumemanager/metrics/metrics_test.go b/pkg/kubelet/volumemanager/metrics/metrics_test.go index cc3f93b166f..20228115f4f 100644 --- a/pkg/kubelet/volumemanager/metrics/metrics_test.go +++ b/pkg/kubelet/volumemanager/metrics/metrics_test.go @@ -19,7 +19,7 @@ package metrics import ( "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" @@ -56,7 +56,7 @@ func TestMetricCollection(t *testing.T) { podName := util.GetUniquePodName(pod) // Add one volume to DesiredStateOfWorld - generatedVolumeName, err := dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "") + generatedVolumeName, err := dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "", "" /* seLinuxLabel */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go index d2d82f4a591..7b0c7536356 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go @@ -283,7 +283,7 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes( } allVolumesAdded := true - mounts, devices := util.GetPodVolumeNames(pod) + mounts, devices, seLinuxContainerContexts := util.GetPodVolumeNames(pod) // Process volume spec for each volume defined in pod for _, podVolume := range pod.Spec.Volumes { @@ -304,7 +304,7 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes( // Add volume to desired state of world uniqueVolumeName, err := dswp.desiredStateOfWorld.AddPodToVolume( - uniquePodName, pod, volumeSpec, podVolume.Name, volumeGidValue) + uniquePodName, pod, volumeSpec, podVolume.Name, volumeGidValue, seLinuxContainerContexts[podVolume.Name]) if err != nil { klog.ErrorS(err, "Failed to add volume to desiredStateOfWorld", "pod", klog.KObj(pod), "volumeName", podVolume.Name, "volumeSpecName", volumeSpec.Name()) dswp.desiredStateOfWorld.AddErrorToPod(uniquePodName, err.Error()) diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 4c33fdf636a..9ea0e961f40 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -199,7 +199,7 @@ func (rc *reconciler) reconcile() { func (rc *reconciler) unmountVolumes() { // Ensure volumes that should be unmounted are unmounted. for _, mountedVolume := range rc.actualStateOfWorld.GetAllMountedVolumes() { - if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName) { + if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName, mountedVolume.SELinuxMountContext) { // Volume is mounted, unmount it klog.V(5).InfoS(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) err := rc.operationExecutor.UnmountVolume( @@ -217,9 +217,14 @@ func (rc *reconciler) unmountVolumes() { func (rc *reconciler) mountOrAttachVolumes() { // Ensure volumes that should be attached/mounted are attached/mounted. for _, volumeToMount := range rc.desiredStateOfWorld.GetVolumesToMount() { - volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, volumeToMount.PersistentVolumeSize) + volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, volumeToMount.PersistentVolumeSize, volumeToMount.SELinuxLabel) volumeToMount.DevicePath = devicePath - if cache.IsVolumeNotAttachedError(err) { + if cache.IsSELinuxMountMismatchError(err) { + // TODO: check error message + lower frequency, this can be noisy + klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("mount precondition failed", err).Error(), "pod", klog.KObj(volumeToMount.Pod)) + // TODO: report error better, this may be too noisy + rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error()) + } else if cache.IsVolumeNotAttachedError(err) { rc.waitForVolumeAttach(volumeToMount) } else if !volMounted || cache.IsRemountRequiredError(err) { rc.mountAttachedVolumes(volumeToMount, err) @@ -373,7 +378,7 @@ func (rc *reconciler) waitForVolumeAttach(volumeToMount cache.VolumeToMount) { func (rc *reconciler) unmountDetachDevices() { for _, attachedVolume := range rc.actualStateOfWorld.GetUnmountedVolumes() { // Check IsOperationPending to avoid marking a volume as detached if it's in the process of mounting. - if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName) && + if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName, attachedVolume.SELinuxMountContext) && !rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName, nestedpendingoperations.EmptyNodeName) { if attachedVolume.DeviceMayBeMounted() { // Volume is globally mounted to device, unmount it @@ -765,7 +770,8 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*gl klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName) continue } - err = rc.actualStateOfWorld.MarkDeviceAsMounted(gvl.volumeName, gvl.devicePath, deviceMountPath) + // TODO(jsafrane): add reconstructed SELinux context + err = rc.actualStateOfWorld.MarkDeviceAsMounted(gvl.volumeName, gvl.devicePath, deviceMountPath, "") if err != nil { klog.ErrorS(err, "Could not mark device is mounted to actual state of world", "volume", gvl.volumeName) continue diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go index c69004e03cc..9a54b878b53 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go @@ -160,7 +160,7 @@ func Test_Run_Positive_VolumeAttachAndMount(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -349,7 +349,7 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabled(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) // Assert @@ -428,7 +428,7 @@ func Test_Run_Negative_VolumeMountControllerAttachEnabled(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -506,7 +506,7 @@ func Test_Run_Positive_VolumeAttachMountUnmountDetach(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -608,7 +608,7 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -715,7 +715,7 @@ func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { } podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -827,7 +827,7 @@ func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) // Assert @@ -924,7 +924,7 @@ func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -1048,7 +1048,7 @@ func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) { podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { @@ -1319,7 +1319,7 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { volumeSpec := &volume.Spec{PersistentVolume: pv} podName := util.GetUniquePodName(pod) volumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) @@ -1340,7 +1340,7 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { // Simulate what DSOWP does pvWithSize.Spec.Capacity[v1.ResourceStorage] = tc.newPVSize volumeSpec = &volume.Spec{PersistentVolume: pvWithSize} - dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) t.Logf("Changing size of the volume to %s", tc.newPVSize.String()) newSize := tc.newPVSize.DeepCopy() @@ -1573,7 +1573,7 @@ func Test_UncertainDeviceGlobalMounts(t *testing.T) { volumeSpec := &volume.Spec{PersistentVolume: pv} podName := util.GetUniquePodName(pod) volumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) @@ -1795,7 +1795,7 @@ func Test_UncertainVolumeMountState(t *testing.T) { volumeSpec := &volume.Spec{PersistentVolume: pv} podName := util.GetUniquePodName(pod) volumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) // Assert if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) @@ -2133,7 +2133,7 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabledRace(t *testing.T) { volumeSpecCopy := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName}) if err != nil { @@ -2158,7 +2158,7 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabledRace(t *testing.T) { klog.InfoS("UnmountDevice called") var generatedVolumeNameCopy v1.UniqueVolumeName generatedVolumeNameCopy, err = dsw.AddPodToVolume( - podName, pod, volumeSpecCopy, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpecCopy, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeNameCopy}) return nil } @@ -2410,7 +2410,7 @@ func TestSyncStates(t *testing.T) { volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} podName := util.GetUniquePodName(pod) volumeName, err := rcInstance.desiredStateOfWorld.AddPodToVolume( - podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */) + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* SELinuxContext */) if err != nil { t.Fatalf("error adding volume %s to dsow: %v", volumeSpec.Name(), err) } diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 03c2abd66bc..0357ea0f5d1 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -536,7 +536,7 @@ func filterUnmountedVolumes(mountedVolumes sets.String, expectedVolumes []string // getExpectedVolumes returns a list of volumes that must be mounted in order to // consider the volume setup step for this pod satisfied. func getExpectedVolumes(pod *v1.Pod) []string { - mounts, devices := util.GetPodVolumeNames(pod) + mounts, devices, _ := util.GetPodVolumeNames(pod) return mounts.Union(devices).UnsortedList() } diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 13079bf1581..d16e287be68 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -166,7 +166,7 @@ func NewOperationExecutor( } } -// MarkVolumeOpts is an struct to pass arguments to MountVolume functions +// MarkVolumeOpts is a struct to pass arguments to MountVolume functions type MarkVolumeOpts struct { PodName volumetypes.UniquePodName PodUID types.UID @@ -177,6 +177,7 @@ type MarkVolumeOpts struct { VolumeGidVolume string VolumeSpec *volume.Spec VolumeMountState VolumeMountState + SELinuxMountContext string } // ActualStateOfWorldMounterUpdater defines a set of operations updating the actual @@ -192,10 +193,10 @@ type ActualStateOfWorldMounterUpdater interface { MarkVolumeMountAsUncertain(markVolumeOpts MarkVolumeOpts) error // Marks the specified volume as having been globally mounted. - MarkDeviceAsMounted(volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error + MarkDeviceAsMounted(volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error // MarkDeviceAsUncertain marks device state in global mount path as uncertain - MarkDeviceAsUncertain(volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error + MarkDeviceAsUncertain(volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error // Marks the specified volume as having its global mount unmounted. MarkDeviceAsUnmounted(volumeName v1.UniqueVolumeName) error @@ -446,6 +447,9 @@ type VolumeToMount struct { // PersistentVolumeSize stores desired size of the volume. // usually this is the size if pv.Spec.Capacity PersistentVolumeSize resource.Quantity + + // SELinux label that should be used to mount. + SELinuxLabel string } // DeviceMountState represents device mount state in a global path. @@ -552,6 +556,8 @@ type AttachedVolume struct { // PluginName is the Unescaped Qualified name of the volume plugin used to // attach and mount this volume. PluginName string + + SELinuxMountContext string } // GenerateMsgDetailed returns detailed msgs for attached volumes @@ -728,6 +734,10 @@ type MountedVolume struct { // DeviceMountPath contains the path on the node where the device should // be mounted after it is attached. DeviceMountPath string + + // SELinuxMountContext is value of mount option 'mount -o context=XYZ'. + // If empty, no such mount option was used. + SELinuxMountContext string } // GenerateMsgDetailed returns detailed msgs for mounted volumes diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 6237755bbdb..cec5ed9a246 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -645,7 +645,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( volumeToMount.VolumeSpec, devicePath, deviceMountPath, - volume.DeviceMounterArgs{FsGroup: fsGroup}, + volume.DeviceMounterArgs{FsGroup: fsGroup, SELinuxLabel: volumeToMount.SELinuxLabel}, ) if err != nil { og.checkForFailedMount(volumeToMount, err) @@ -659,7 +659,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( // Update actual state of world to reflect volume is globally mounted markDeviceMountedErr := actualStateOfWorld.MarkDeviceAsMounted( - volumeToMount.VolumeName, devicePath, deviceMountPath) + volumeToMount.VolumeName, devicePath, deviceMountPath, volumeToMount.SELinuxLabel) if markDeviceMountedErr != nil { // On failure, return error. Caller will log and retry. eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr) @@ -688,6 +688,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( FsGroup: hostGID, DesiredSize: volumeToMount.DesiredSizeLimit, FSGroupChangePolicy: fsGroupChangePolicy, + SELinuxLabel: volumeToMount.SELinuxLabel, }) // Update actual state of world markOpts := MarkVolumeOpts{ @@ -699,6 +700,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( VolumeGidVolume: volumeToMount.VolumeGidValue, VolumeSpec: volumeToMount.VolumeSpec, VolumeMountState: VolumeMounted, + SELinuxMountContext: volumeToMount.SELinuxLabel, } if mountErr != nil { og.checkForFailedMount(volumeToMount, mountErr) @@ -787,7 +789,7 @@ func (og *operationGenerator) markDeviceErrorState(volumeToMount VolumeToMount, actualStateOfWorld.GetDeviceMountState(volumeToMount.VolumeName) == DeviceNotMounted { // only devices which are not mounted can be marked as uncertain. We do not want to mark a device // which was previously marked as mounted here as uncertain. - markDeviceUncertainError := actualStateOfWorld.MarkDeviceAsUncertain(volumeToMount.VolumeName, devicePath, deviceMountPath) + markDeviceUncertainError := actualStateOfWorld.MarkDeviceAsUncertain(volumeToMount.VolumeName, devicePath, deviceMountPath, volumeToMount.SELinuxLabel) if markDeviceUncertainError != nil { klog.Errorf(volumeToMount.GenerateErrorDetailed("MountDevice.MarkDeviceAsUncertain failed", markDeviceUncertainError).Error()) } @@ -955,7 +957,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( unmountDeviceErr := volumeDeviceUnmounter.UnmountDevice(deviceMountPath) if unmountDeviceErr != nil { // Mark the device as uncertain, so MountDevice is called for new pods. UnmountDevice may be already in progress. - markDeviceUncertainErr := actualStateOfWorld.MarkDeviceAsUncertain(deviceToDetach.VolumeName, deviceToDetach.DevicePath, deviceMountPath) + markDeviceUncertainErr := actualStateOfWorld.MarkDeviceAsUncertain(deviceToDetach.VolumeName, deviceToDetach.DevicePath, deviceMountPath, deviceToDetach.SELinuxMountContext) if markDeviceUncertainErr != nil { // There is nothing else we can do. Hope that UnmountDevice will be re-tried shortly. klog.Errorf(deviceToDetach.GenerateErrorDetailed("UnmountDevice.MarkDeviceAsUncertain failed", markDeviceUncertainErr).Error()) @@ -976,7 +978,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( // The device is still in use elsewhere. Caller will log and retry. if deviceOpened { // Mark the device as uncertain, so MountDevice is called for new pods. - markDeviceUncertainErr := actualStateOfWorld.MarkDeviceAsUncertain(deviceToDetach.VolumeName, deviceToDetach.DevicePath, deviceMountPath) + markDeviceUncertainErr := actualStateOfWorld.MarkDeviceAsUncertain(deviceToDetach.VolumeName, deviceToDetach.DevicePath, deviceMountPath, deviceToDetach.SELinuxMountContext) if markDeviceUncertainErr != nil { // There is nothing else we can do. Hope that UnmountDevice will be re-tried shortly. klog.Errorf(deviceToDetach.GenerateErrorDetailed("UnmountDevice.MarkDeviceAsUncertain failed", markDeviceUncertainErr).Error()) @@ -1114,7 +1116,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( // Update actual state of world to reflect volume is globally mounted markedDevicePath := devicePath markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( - volumeToMount.VolumeName, markedDevicePath, globalMapPath) + volumeToMount.VolumeName, markedDevicePath, globalMapPath, "") if markDeviceMappedErr != nil { // On failure, return error. Caller will log and retry. eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr) @@ -1183,7 +1185,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( // TODO: This can be improved after #82492 is merged and ASW has state. if markedDevicePath != devicePath { markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( - volumeToMount.VolumeName, devicePath, globalMapPath) + volumeToMount.VolumeName, devicePath, globalMapPath, "") if markDeviceMappedErr != nil { // On failure, return error. Caller will log and retry. eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr) @@ -1411,7 +1413,7 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( // cases below. The volume is marked as fully un-mapped at the end of this function, when everything // succeeds. markDeviceUncertainErr := actualStateOfWorld.MarkDeviceAsUncertain( - deviceToDetach.VolumeName, deviceToDetach.DevicePath, globalMapPath) + deviceToDetach.VolumeName, deviceToDetach.DevicePath, globalMapPath, "" /* seLinuxMountContext */) if markDeviceUncertainErr != nil { // On failure, return error. Caller will log and retry. eventErr, detailedErr := deviceToDetach.GenerateError("UnmapDevice.MarkDeviceAsUncertain failed", markDeviceUncertainErr) diff --git a/pkg/volume/util/selinux.go b/pkg/volume/util/selinux.go new file mode 100644 index 00000000000..f05c30121f4 --- /dev/null +++ b/pkg/volume/util/selinux.go @@ -0,0 +1,117 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "github.com/opencontainers/selinux/go-selinux" + "github.com/opencontainers/selinux/go-selinux/label" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/volume" +) + +// SELinuxOptionsToFileLabel returns SELinux file label for given options. +func SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) { + if opts == nil { + return "", nil + } + + args := contextOptions(opts) + if len(args) == 0 { + return "", nil + } + + // TODO: use interface for InitLabels for unit tests. + processLabel, fileLabel, err := label.InitLabels(args) + if err != nil { + // In theory, this should be unreachable. InitLabels can fail only when args contain an unknown option, + // and all options returned by contextOptions are known. + return "", err + } + // InitLabels() may allocate a new unique SELinux label in kubelet memory. The label is *not* allocated + // in the container runtime. Clear it to avoid memory problems. + // ReleaseLabel on non-allocated label is NOOP. + selinux.ReleaseLabel(processLabel) + + return fileLabel, nil +} + +// Convert SELinuxOptions to []string accepted by label.InitLabels +func contextOptions(opts *v1.SELinuxOptions) []string { + if opts == nil { + return nil + } + args := make([]string, 0, 3) + if opts.User != "" { + args = append(args, "user:"+opts.User) + } + if opts.Role != "" { + args = append(args, "role:"+opts.Role) + } + if opts.Type != "" { + args = append(args, "type:"+opts.Type) + } + if opts.Level != "" { + args = append(args, "level:"+opts.Level) + } + return args +} + +// SupportsSELinuxContextMount checks if the given volumeSpec supports with mount -o context +func SupportsSELinuxContextMount(volumeSpec *volume.Spec, volumePluginMgr *volume.VolumePluginMgr) (bool, error) { + // This is cheap + if !selinux.GetEnabled() { + return false, nil + } + + plugin, _ := volumePluginMgr.FindPluginBySpec(volumeSpec) + if plugin != nil { + return plugin.SupportsSELinuxContextMount(volumeSpec) + } + + return false, nil +} + +func IsRWOP(volumeSpec *volume.Spec) bool { + if !utilfeature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod) { + return false + } + if volumeSpec.PersistentVolume == nil { + return false + } + if len(volumeSpec.PersistentVolume.Spec.AccessModes) != 1 { + return false + } + if !v1helper.ContainsAccessMode(volumeSpec.PersistentVolume.Spec.AccessModes, v1.ReadWriteOncePod) { + return false + } + return true +} + +// AddSELinuxMountOption adds -o context="XYZ" mount option to a given list +func AddSELinuxMountOption(options []string, seLinuxContext string) []string { + if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + return options + } + // Use double quotes to support a comma "," in the SELinux context string. + // For example: dirsync,context="system_u:object_r:container_file_t:s0:c15,c25",noatime + return append(options, fmt.Sprintf("context=%q", seLinuxContext)) +} diff --git a/pkg/volume/util/util.go b/pkg/volume/util/util.go index dae2ec59c37..e7df9538f32 100644 --- a/pkg/volume/util/util.go +++ b/pkg/volume/util/util.go @@ -275,16 +275,6 @@ func JoinMountOptions(userOptions []string, systemOptions []string) []string { return allMountOptions.List() } -// AddSELinuxMountOption adds -o context="XYZ mount option to a given list -func AddSELinuxMountOption(options []string, seLinuxContext string) []string { - if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { - return options - } - // Use double quotes to support a comma "," in the SELinux context string. - // For example: dirsync,context="system_u:object_r:container_file_t:s0:c15,c25",noatime - return append(options, "context=%q", seLinuxContext) -} - // ContainsAccessMode returns whether the requested mode is contained by modes func ContainsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool { for _, m := range modes { @@ -584,15 +574,29 @@ func IsLocalEphemeralVolume(volume v1.Volume) bool { } // GetPodVolumeNames returns names of volumes that are used in a pod, -// either as filesystem mount or raw block device. -func GetPodVolumeNames(pod *v1.Pod) (mounts sets.String, devices sets.String) { +// either as filesystem mount or raw block device, together with list +// of all SELinux contexts of all containers that use the volumes. +func GetPodVolumeNames(pod *v1.Pod) (mounts sets.String, devices sets.String, seLinuxContainerContexts map[string][]*v1.SELinuxOptions) { mounts = sets.NewString() devices = sets.NewString() + seLinuxContainerContexts = make(map[string][]*v1.SELinuxOptions) podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(container *v1.Container, containerType podutil.ContainerType) bool { + var seLinuxOptions *v1.SELinuxOptions + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + effectiveContainerSecurity := securitycontext.DetermineEffectiveSecurityContext(pod, container) + if effectiveContainerSecurity != nil { + // No DeepCopy, SELinuxOptions is already a copy of Pod's or container's SELinuxOptions + seLinuxOptions = effectiveContainerSecurity.SELinuxOptions + } + } + if container.VolumeMounts != nil { for _, mount := range container.VolumeMounts { mounts.Insert(mount.Name) + if seLinuxOptions != nil { + seLinuxContainerContexts[mount.Name] = append(seLinuxContainerContexts[mount.Name], seLinuxOptions.DeepCopy()) + } } } if container.VolumeDevices != nil { From b2e18c0b20895012a3002bd0f6ee96f11d0656c1 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 18 Mar 2022 14:37:12 +0100 Subject: [PATCH 09/22] Add metrics for SELinux context mount Add separate _errors and _warnings to capture volumes that were rejected from those will be rejected when the feature is expanded to all access mode. --- .../desired_state_of_wold_selinux_metrics.go | 84 +++++++++++++++++++ .../cache/desired_state_of_world.go | 19 ++++- 2 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 pkg/kubelet/volumemanager/cache/desired_state_of_wold_selinux_metrics.go diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_wold_selinux_metrics.go b/pkg/kubelet/volumemanager/cache/desired_state_of_wold_selinux_metrics.go new file mode 100644 index 00000000000..16727d02591 --- /dev/null +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_wold_selinux_metrics.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "sync" + + compbasemetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var ( + // TODO: add plugin name + access mode labels to all these metrics + seLinuxContainerContextErrors = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_container_errors_total", + Help: "Number of errors when kubelet cannot compute SELinux context for a container. Kubelet can't start such a Pod then and it will retry, therefore value of this metric may not represent the actual nr. of containers.", + StabilityLevel: compbasemetrics.ALPHA, + }) + seLinuxContainerContextWarnings = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_container_warnings_total", + StabilityLevel: compbasemetrics.ALPHA, + Help: "Number of errors when kubelet cannot compute SELinux context for a container that are ignored. They will become real errors when SELinuxMountReadWriteOncePod feature is expanded to all volume access modes.", + }) + seLinuxPodContextMismatchErrors = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_pod_context_mismatch_errors_total", + Help: "Number of errors when a Pod defines different SELinux contexts for its containers that use the same volume. Kubelet can't start such a Pod then and it will retry, therefore value of this metric may not represent the actual nr. of Pods.", + StabilityLevel: compbasemetrics.ALPHA, + }) + seLinuxPodContextMismatchWarnings = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_pod_context_mismatch_warnings_total", + Help: "Number of errors when a Pod defines different SELinux contexts for its containers that use the same volume. They are not errors yet, but they will become real errors when SELinuxMountReadWriteOncePod feature is expanded to all volume access modes.", + StabilityLevel: compbasemetrics.ALPHA, + }) + seLinuxVolumeContextMismatchErrors = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_volume_context_mismatch_errors_total", + Help: "Number of errors when a Pod uses a volume that is already mounted with a different SELinux context than the Pod needs. Kubelet can't start such a Pod then and it will retry, therefore value of this metric may not represent the actual nr. of Pods.", + StabilityLevel: compbasemetrics.ALPHA, + }) + seLinuxVolumeContextMismatchWarnings = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_volume_context_mismatch_warnings_total", + Help: "Number of errors when a Pod uses a volume that is already mounted with a different SELinux context than the Pod needs. They are not errors yet, but they will become real errors when SELinuxMountReadWriteOncePod feature is expanded to all volume access modes.", + StabilityLevel: compbasemetrics.ALPHA, + }) + seLinuxVolumesAdmitted = compbasemetrics.NewGauge( + &compbasemetrics.GaugeOpts{ + Name: "volume_manager_selinux_volumes_admitted_total", + Help: "Number of volumes whose SELinux context was fine and will be mounted with mount -o context option.", + StabilityLevel: compbasemetrics.ALPHA, + }) + + registerMetrics sync.Once +) + +func registerSELinuxMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister(seLinuxContainerContextErrors) + legacyregistry.MustRegister(seLinuxContainerContextWarnings) + legacyregistry.MustRegister(seLinuxPodContextMismatchErrors) + legacyregistry.MustRegister(seLinuxPodContextMismatchWarnings) + legacyregistry.MustRegister(seLinuxVolumeContextMismatchErrors) + legacyregistry.MustRegister(seLinuxVolumeContextMismatchWarnings) + legacyregistry.MustRegister(seLinuxVolumesAdmitted) + }) +} diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 148f3775ca6..3b7f6c35113 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -142,6 +142,9 @@ type VolumeToMount struct { // NewDesiredStateOfWorld returns a new instance of DesiredStateOfWorld. func NewDesiredStateOfWorld(volumePluginMgr *volume.VolumePluginMgr) DesiredStateOfWorld { + if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + registerSELinuxMetrics() + } return &desiredStateOfWorld{ volumesToMount: make(map[v1.UniqueVolumeName]volumeToMount), volumePluginMgr: volumePluginMgr, @@ -302,10 +305,11 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) if isRWOP { // Cannot mount with -o context if the context can't be composed. + seLinuxContainerContextErrors.Add(1.0) return "", fullErr } else { // This is not an error yet, but it will be when support for RWO and RWX volumes is added - // TODO: bump some metric here + seLinuxContainerContextWarnings.Add(1.0) klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") break } @@ -317,10 +321,11 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( if seLinuxFileLabel != newLabel { fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) if isRWOP { + seLinuxPodContextMismatchErrors.Add(1.0) return "", fullErr } else { // This is not an error yet, but it will be when support for RWO and RWX volumes is added - // TODO: bump some metric here + seLinuxPodContextMismatchWarnings.Add(1.0) klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") break } @@ -350,6 +355,9 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( } } } + if seLinuxFileLabel != "" { + seLinuxVolumesAdmitted.Add(1.0) + } vmt := volumeToMount{ volumeName: volumeName, podsToMount: make(map[types.UniquePodName]podToMount), @@ -376,12 +384,17 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( // TODO: update the error message after tests, e.g. add at least the conflicting pod names. fullErr := fmt.Errorf("conflicting SELinux labels of volume %s: %q and %q", volumeSpec.Name(), vol.seLinuxFileLabel, seLinuxFileLabel) if isRWOP { + seLinuxVolumeContextMismatchErrors.Add(1.0) return "", fullErr } else { // This is not an error yet, but it will be when support for RWO and RWX volumes is added - // TODO: bump some metric here + seLinuxVolumeContextMismatchWarnings.Add(1.0) klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") } + } else { + if seLinuxFileLabel != "" { + seLinuxVolumesAdmitted.Add(1.0) + } } } } From de7f5b66eda11d0d481acf696bd927fbbd53f8a3 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 21 Mar 2022 11:05:37 +0100 Subject: [PATCH 10/22] Fix existing unit tests --- .../cache/desired_state_of_world_test.go | 54 ++++++++++--------- .../volumemanager/metrics/metrics_test.go | 2 +- .../desired_state_of_world_populator_test.go | 44 +++++++-------- .../reconciler/reconciler_test.go | 16 ++++-- pkg/volume/util/util_test.go | 2 +- 5 files changed, 63 insertions(+), 55 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go index 3d3d6639976..593228ef42c 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go @@ -67,10 +67,10 @@ func Test_AddPodToVolume_Positive_NewPodNewVolume(t *testing.T) { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } - verifyVolumeExistsDsw(t, generatedVolumeName, dsw) + verifyVolumeExistsDsw(t, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolumeName, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) } @@ -112,10 +112,10 @@ func Test_AddPodToVolume_Positive_ExistingPodExistingVolume(t *testing.T) { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } - verifyVolumeExistsDsw(t, generatedVolumeName, dsw) + verifyVolumeExistsDsw(t, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolumeName, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) } @@ -314,18 +314,18 @@ func Test_DeletePodFromVolume_Positive_PodExistsVolumeExists(t *testing.T) { if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } - verifyVolumeExistsDsw(t, generatedVolumeName, dsw) + verifyVolumeExistsDsw(t, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolumeName, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinuxContext */, dsw) // Act dsw.DeletePodFromVolume(podName, generatedVolumeName) // Assert - verifyVolumeDoesntExist(t, generatedVolumeName, dsw) + verifyVolumeDoesntExist(t, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeDoesntExistInVolumesToMount(t, generatedVolumeName, dsw) - verifyPodDoesntExistInVolumeDsw(t, podName, generatedVolumeName, dsw) + verifyPodDoesntExistInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinuxContext */, dsw) verifyVolumeDoesntExistWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) } @@ -430,36 +430,36 @@ func Test_MarkVolumesReportedInUse_Positive_NewPodNewVolume(t *testing.T) { dsw.MarkVolumesReportedInUse(volumesReportedInUse) // Assert - verifyVolumeExistsDsw(t, generatedVolume1Name, dsw) + verifyVolumeExistsDsw(t, generatedVolume1Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume1Name, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod1Name, generatedVolume1Name, dsw) - verifyVolumeExistsDsw(t, generatedVolume2Name, dsw) + verifyPodExistsInVolumeDsw(t, pod1Name, generatedVolume1Name, "" /* SELinuxContext */, dsw) + verifyVolumeExistsDsw(t, generatedVolume2Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume2Name, true /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolume2Name, dsw) - verifyVolumeExistsDsw(t, generatedVolume3Name, dsw) + verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolume2Name, "" /* SELinuxContext */, dsw) + verifyVolumeExistsDsw(t, generatedVolume3Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume3Name, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod3Name, generatedVolume3Name, dsw) + verifyPodExistsInVolumeDsw(t, pod3Name, generatedVolume3Name, "" /* SELinuxContext */, dsw) // Act volumesReportedInUse = []v1.UniqueVolumeName{generatedVolume3Name} dsw.MarkVolumesReportedInUse(volumesReportedInUse) // Assert - verifyVolumeExistsDsw(t, generatedVolume1Name, dsw) + verifyVolumeExistsDsw(t, generatedVolume1Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume1Name, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod1Name, generatedVolume1Name, dsw) - verifyVolumeExistsDsw(t, generatedVolume2Name, dsw) + verifyPodExistsInVolumeDsw(t, pod1Name, generatedVolume1Name, "" /* SELinuxContext */, dsw) + verifyVolumeExistsDsw(t, generatedVolume2Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume2Name, false /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolume2Name, dsw) - verifyVolumeExistsDsw(t, generatedVolume3Name, dsw) + verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolume2Name, "" /* SELinuxContext */, dsw) + verifyVolumeExistsDsw(t, generatedVolume3Name, "" /* SELinuxContext */, dsw) verifyVolumeExistsInVolumesToMount( t, generatedVolume3Name, true /* expectReportedInUse */, dsw) - verifyPodExistsInVolumeDsw(t, pod3Name, generatedVolume3Name, dsw) + verifyPodExistsInVolumeDsw(t, pod3Name, generatedVolume3Name, "" /* SELinuxContext */, dsw) } func Test_AddPodToVolume_WithEmptyDirSizeLimit(t *testing.T) { @@ -598,8 +598,8 @@ func Test_AddPodToVolume_WithEmptyDirSizeLimit(t *testing.T) { } func verifyVolumeExistsDsw( - t *testing.T, expectedVolumeName v1.UniqueVolumeName, dsw DesiredStateOfWorld) { - volumeExists := dsw.VolumeExists(expectedVolumeName) + t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectedSELinuxContext string, dsw DesiredStateOfWorld) { + volumeExists := dsw.VolumeExists(expectedVolumeName, expectedSELinuxContext) if !volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -609,8 +609,8 @@ func verifyVolumeExistsDsw( } func verifyVolumeDoesntExist( - t *testing.T, expectedVolumeName v1.UniqueVolumeName, dsw DesiredStateOfWorld) { - volumeExists := dsw.VolumeExists(expectedVolumeName) + t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectedSELinuxContext string, dsw DesiredStateOfWorld) { + volumeExists := dsw.VolumeExists(expectedVolumeName, expectedSELinuxContext) if volumeExists { t.Fatalf( "VolumeExists(%q) returned incorrect value. Expected: Actual: <%v>", @@ -661,9 +661,10 @@ func verifyPodExistsInVolumeDsw( t *testing.T, expectedPodName volumetypes.UniquePodName, expectedVolumeName v1.UniqueVolumeName, + expectedSeLinuxContext string, dsw DesiredStateOfWorld) { if podExistsInVolume := dsw.PodExistsInVolume( - expectedPodName, expectedVolumeName); !podExistsInVolume { + expectedPodName, expectedVolumeName, expectedSeLinuxContext); !podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -674,9 +675,10 @@ func verifyPodDoesntExistInVolumeDsw( t *testing.T, expectedPodName volumetypes.UniquePodName, expectedVolumeName v1.UniqueVolumeName, + expectedSeLinuxContext string, dsw DesiredStateOfWorld) { if podExistsInVolume := dsw.PodExistsInVolume( - expectedPodName, expectedVolumeName); podExistsInVolume { + expectedPodName, expectedVolumeName, expectedSeLinuxContext); podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) diff --git a/pkg/kubelet/volumemanager/metrics/metrics_test.go b/pkg/kubelet/volumemanager/metrics/metrics_test.go index 20228115f4f..cd30a8023c1 100644 --- a/pkg/kubelet/volumemanager/metrics/metrics_test.go +++ b/pkg/kubelet/volumemanager/metrics/metrics_test.go @@ -56,7 +56,7 @@ func TestMetricCollection(t *testing.T) { podName := util.GetUniquePodName(pod) // Add one volume to DesiredStateOfWorld - generatedVolumeName, err := dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "", "" /* seLinuxLabel */) + generatedVolumeName, err := dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "", nil /* seLinuxOptions */) if err != nil { t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) } diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index ea586756087..24769116004 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -243,7 +243,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { t.Fatalf("Failed to remove pods from desired state of world since they no longer exist") } - volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName) + volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -252,7 +252,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) { } if podExistsInVolume := dswp.desiredStateOfWorld.PodExistsInVolume( - podName, expectedVolumeName); podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -283,7 +283,7 @@ func TestFindAndRemoveDeletedPodsWithActualState(t *testing.T) { dswp.findAndRemoveDeletedPods() // Although Pod status is terminated, pod still exists in pod manager and actual state does not has this pod and volume information // desired state populator will fail to delete this pod and volume first - volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName) + volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if !volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -292,7 +292,7 @@ func TestFindAndRemoveDeletedPodsWithActualState(t *testing.T) { } if podExistsInVolume := dswp.desiredStateOfWorld.PodExistsInVolume( - podName, expectedVolumeName); !podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); !podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -302,7 +302,7 @@ func TestFindAndRemoveDeletedPodsWithActualState(t *testing.T) { // desired state populator now can successfully delete the pod and volume reconcileASW(fakeASW, dswp.desiredStateOfWorld, t) dswp.findAndRemoveDeletedPods() - if !dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName) { + if !dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", expectedVolumeName, @@ -315,7 +315,7 @@ func TestFindAndRemoveDeletedPodsWithActualState(t *testing.T) { // desired state populator now can successfully delete the pod and volume reconcileASW(fakeASW, dswp.desiredStateOfWorld, t) dswp.findAndRemoveDeletedPods() - volumeExists = dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName) + volumeExists = dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -324,7 +324,7 @@ func TestFindAndRemoveDeletedPodsWithActualState(t *testing.T) { } if podExistsInVolume := dswp.desiredStateOfWorld.PodExistsInVolume( - podName, expectedVolumeName); podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -369,7 +369,7 @@ func TestFindAndRemoveDeletedPodsWithUncertain(t *testing.T) { t.Fatalf("Failed to remove pods from desired state of world since they no longer exist") } - volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName) + volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -378,7 +378,7 @@ func TestFindAndRemoveDeletedPodsWithUncertain(t *testing.T) { } if podExistsInVolume := dswp.desiredStateOfWorld.PodExistsInVolume( - podName, expectedVolumeName); podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -443,7 +443,7 @@ func prepareDSWPWithPodPV(t *testing.T) (*desiredStateOfWorldPopulator, *fakePod expectedVolumeName := v1.UniqueVolumeName(generatedVolumeName) - volumeExists := fakesDSW.VolumeExists(expectedVolumeName) + volumeExists := fakesDSW.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if !volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -452,7 +452,7 @@ func prepareDSWPWithPodPV(t *testing.T) (*desiredStateOfWorldPopulator, *fakePod } if podExistsInVolume := fakesDSW.PodExistsInVolume( - podName, expectedVolumeName); !podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); !podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -514,7 +514,7 @@ func TestFindAndRemoveNonattachableVolumes(t *testing.T) { expectedVolumeName := v1.UniqueVolumeName(generatedVolumeName) - volumeExists := fakesDSW.VolumeExists(expectedVolumeName) + volumeExists := fakesDSW.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if !volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -614,7 +614,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t expectedVolumeName := v1.UniqueVolumeName(generatedVolumeName) - volumeExists := fakesDSW.VolumeExists(expectedVolumeName) + volumeExists := fakesDSW.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if !volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -623,7 +623,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t } if podExistsInVolume := fakesDSW.PodExistsInVolume( - podName, expectedVolumeName); !podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); !podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -647,7 +647,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t t.Fatalf("Failed to remove pods from desired state of world since they no longer exist") } - volumeExists = fakesDSW.VolumeExists(expectedVolumeName) + volumeExists = fakesDSW.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if volumeExists { t.Fatalf( "VolumeExists(%q) failed. Expected: Actual: <%v>", @@ -656,7 +656,7 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t } if podExistsInVolume := fakesDSW.PodExistsInVolume( - podName, expectedVolumeName); podExistsInVolume { + podName, expectedVolumeName, "" /* SELinuxContext */); podExistsInVolume { t.Fatalf( "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", podExistsInVolume) @@ -709,7 +709,7 @@ func TestCreateVolumeSpec_Valid_File_VolumeMounts(t *testing.T) { pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) - mountsMap, devicesMap := util.GetPodVolumeNames(pod) + mountsMap, devicesMap, _ := util.GetPodVolumeNames(pod) _, volumeSpec, _, err := dswp.createVolumeSpec(pod.Spec.Volumes[0], pod, mountsMap, devicesMap) @@ -755,7 +755,7 @@ func TestCreateVolumeSpec_Valid_Nil_VolumeMounts(t *testing.T) { pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) - mountsMap, devicesMap := util.GetPodVolumeNames(pod) + mountsMap, devicesMap, _ := util.GetPodVolumeNames(pod) _, volumeSpec, _, err := dswp.createVolumeSpec(pod.Spec.Volumes[0], pod, mountsMap, devicesMap) @@ -801,7 +801,7 @@ func TestCreateVolumeSpec_Valid_Block_VolumeDevices(t *testing.T) { pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers) fakePodManager.AddPod(pod) - mountsMap, devicesMap := util.GetPodVolumeNames(pod) + mountsMap, devicesMap, _ := util.GetPodVolumeNames(pod) _, volumeSpec, _, err := dswp.createVolumeSpec(pod.Spec.Volumes[0], pod, mountsMap, devicesMap) @@ -847,7 +847,7 @@ func TestCreateVolumeSpec_Invalid_File_VolumeDevices(t *testing.T) { pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers) fakePodManager.AddPod(pod) - mountsMap, devicesMap := util.GetPodVolumeNames(pod) + mountsMap, devicesMap, _ := util.GetPodVolumeNames(pod) _, volumeSpec, _, err := dswp.createVolumeSpec(pod.Spec.Volumes[0], pod, mountsMap, devicesMap) @@ -893,7 +893,7 @@ func TestCreateVolumeSpec_Invalid_Block_VolumeMounts(t *testing.T) { pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers) fakePodManager.AddPod(pod) - mountsMap, devicesMap := util.GetPodVolumeNames(pod) + mountsMap, devicesMap, _ := util.GetPodVolumeNames(pod) _, volumeSpec, _, err := dswp.createVolumeSpec(pod.Spec.Volumes[0], pod, mountsMap, devicesMap) @@ -1142,7 +1142,7 @@ func reprocess(dswp *desiredStateOfWorldPopulator, uniquePodName types.UniquePod func getResizeRequiredVolumes(dsw cache.DesiredStateOfWorld, asw cache.ActualStateOfWorld, newSize resource.Quantity) []v1.UniqueVolumeName { resizeRequiredVolumes := []v1.UniqueVolumeName{} for _, volumeToMount := range dsw.GetVolumesToMount() { - _, _, err := asw.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, newSize) + _, _, err := asw.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, newSize, "" /* SELinuxContext */) if cache.IsFSResizeRequiredError(err) { resizeRequiredVolumes = append(resizeRequiredVolumes, volumeToMount.VolumeName) } diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go index 9a54b878b53..bd49d90b994 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go @@ -260,7 +260,13 @@ func Test_Run_Positive_VolumeAttachAndMountMigrationEnabled(t *testing.T) { podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( - podName, pod, migratedSpec, migratedSpec.Name(), "" /* volumeGidValue */) + podName, + pod, + migratedSpec, + migratedSpec.Name(), + "", /* volumeGidValue */ + nil, /* SELinuxContexts */ + ) // Assert if err != nil { @@ -1340,13 +1346,13 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { // Simulate what DSOWP does pvWithSize.Spec.Capacity[v1.ResourceStorage] = tc.newPVSize volumeSpec = &volume.Spec{PersistentVolume: pvWithSize} - dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxLabel */) + dsw.AddPodToVolume(podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* seLinuxContexts */) t.Logf("Changing size of the volume to %s", tc.newPVSize.String()) newSize := tc.newPVSize.DeepCopy() dsw.UpdatePersistentVolumeSize(volumeName, &newSize) - _, _, podExistErr := asw.PodExistsInVolume(podName, volumeName, newSize) + _, _, podExistErr := asw.PodExistsInVolume(podName, volumeName, newSize, "" /* SELinuxLabel */) if tc.expansionFailed { if cache.IsFSResizeRequiredError(podExistErr) { t.Fatalf("volume %s should not throw fsResizeRequired error: %v", volumeName, podExistErr) @@ -1358,7 +1364,7 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { go reconciler.Run(wait.NeverStop) waitErr := retryWithExponentialBackOff(testOperationBackOffDuration, func() (done bool, err error) { - mounted, _, err := asw.PodExistsInVolume(podName, volumeName, newSize) + mounted, _, err := asw.PodExistsInVolume(podName, volumeName, newSize, "" /* SELinuxContext */) return mounted && err == nil, nil }) if waitErr != nil { @@ -1913,7 +1919,7 @@ func waitForUncertainPodMount(t *testing.T, volumeName v1.UniqueVolumeName, podN err := retryWithExponentialBackOff( testOperationBackOffDuration, func() (bool, error) { - mounted, _, err := asw.PodExistsInVolume(podName, volumeName, resource.Quantity{}) + mounted, _, err := asw.PodExistsInVolume(podName, volumeName, resource.Quantity{}, "" /* SELinuxContext */) if mounted || err != nil { return false, nil } diff --git a/pkg/volume/util/util_test.go b/pkg/volume/util/util_test.go index 9b7e7559144..0613eafc739 100644 --- a/pkg/volume/util/util_test.go +++ b/pkg/volume/util/util_test.go @@ -785,7 +785,7 @@ func TestGetPodVolumeNames(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - mounts, devices := GetPodVolumeNames(test.pod) + mounts, devices, _ := GetPodVolumeNames(test.pod) if !mounts.Equal(test.expectedMounts) { t.Errorf("Expected mounts: %q, got %q", mounts.List(), test.expectedMounts.List()) } From 5c90474f382956565df9edd4f7f438d96ff88287 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 21 Mar 2022 16:39:46 +0100 Subject: [PATCH 11/22] Add SELinux mount support to CSI driver With some minor refactoring to use common getCSIDriver function. --- pkg/volume/csi/csi_attacher.go | 21 ++++++--- pkg/volume/csi/csi_mounter.go | 33 ++++++++++---- pkg/volume/csi/csi_mounter_test.go | 2 +- pkg/volume/csi/csi_plugin.go | 72 ++++++++++++++---------------- 4 files changed, 75 insertions(+), 53 deletions(-) diff --git a/pkg/volume/csi/csi_attacher.go b/pkg/volume/csi/csi_attacher.go index 11de3c558bc..57fbcb7e9c5 100644 --- a/pkg/volume/csi/csi_attacher.go +++ b/pkg/volume/csi/csi_attacher.go @@ -26,9 +26,6 @@ import ( "strings" "time" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/klog/v2" - v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -36,9 +33,12 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" "k8s.io/utils/clock" ) @@ -331,8 +331,9 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath)) dataDir := filepath.Dir(deviceMountPath) data := map[string]string{ - volDataKey.volHandle: csiSource.VolumeHandle, - volDataKey.driverName: csiSource.Driver, + volDataKey.volHandle: csiSource.VolumeHandle, + volDataKey.driverName: csiSource.Driver, + volDataKey.seLinuxMountContext: deviceMounterArgs.SELinuxLabel, } err = saveVolumeData(dataDir, volDataFileName, data) @@ -371,6 +372,16 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo mountOptions = spec.PersistentVolume.Spec.MountOptions } + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + support, err := c.plugin.SupportsSELinuxContextMount(spec) + if err != nil { + return errors.New(log("failed to query for SELinuxMount support: %s", err)) + } + if support { + mountOptions = util.AddSELinuxMountOption(mountOptions, deviceMounterArgs.SELinuxLabel) + } + } + var nodeStageFSGroupArg *int64 if utilfeature.DefaultFeatureGate.Enabled(features.DelegateFSGroupToCSIDriver) { driverSupportsCSIVolumeMountGroup, err := csi.NodeSupportsVolumeMountGroup(ctx) diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index 1e85796a216..42175153c11 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -24,8 +24,6 @@ import ( "os" "path/filepath" - "k8s.io/klog/v2" - authenticationv1 "k8s.io/api/authentication/v1" api "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" @@ -33,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" @@ -49,7 +48,8 @@ var ( driverName, nodeName, attachmentID, - volumeLifecycleMode string + volumeLifecycleMode, + seLinuxMountContext string }{ "specVolID", "volumeHandle", @@ -57,6 +57,7 @@ var ( "nodeName", "attachmentID", "volumeLifecycleMode", + "seLinuxMountContext", } ) @@ -70,7 +71,7 @@ type csiMountMgr struct { volumeID string specVolumeID string readOnly bool - supportsSELinux bool + needSELinuxRelabel bool spec *volume.Spec pod *api.Pod podUID types.UID @@ -245,6 +246,18 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error } } + var selinuxLabelMount bool + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + support, err := c.plugin.SupportsSELinuxContextMount(c.spec) + if err != nil { + return errors.New(log("failed to query for SELinuxMount support: %s", err)) + } + if support { + mountOptions = util.AddSELinuxMountOption(mountOptions, mounterArgs.SELinuxLabel) + selinuxLabelMount = true + } + } + err = csi.NodePublishVolume( ctx, volumeHandle, @@ -270,10 +283,12 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error return err } - c.supportsSELinux, err = c.kubeVolHost.GetHostUtil().GetSELinuxSupport(dir) - if err != nil { - // The volume is mounted. Return UncertainProgressError, so kubelet will unmount it when user deletes the pod. - return volumetypes.NewUncertainProgressError(fmt.Sprintf("error checking for SELinux support: %s", err)) + if !selinuxLabelMount { + c.needSELinuxRelabel, err = c.kubeVolHost.GetHostUtil().GetSELinuxSupport(dir) + if err != nil { + // The volume is mounted. Return UncertainProgressError, so kubelet will unmount it when user deletes the pod. + return volumetypes.NewUncertainProgressError(fmt.Sprintf("error checking for SELinux support: %s", err)) + } } if !driverSupportsCSIVolumeMountGroup && c.supportsFSGroup(fsType, mounterArgs.FsGroup, c.fsGroupPolicy) { @@ -350,7 +365,7 @@ func (c *csiMountMgr) GetAttributes() volume.Attributes { return volume.Attributes{ ReadOnly: c.readOnly, Managed: !c.readOnly, - SELinuxRelabel: c.supportsSELinux, + SELinuxRelabel: c.needSELinuxRelabel, } } diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index e8f5c5fea38..458643aff05 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -1235,7 +1235,7 @@ func Test_csiMountMgr_supportsFSGroup(t *testing.T) { volumeID: tt.fields.volumeID, specVolumeID: tt.fields.specVolumeID, readOnly: tt.fields.readOnly, - supportsSELinux: tt.fields.supportsSELinux, + needSELinuxRelabel: tt.fields.supportsSELinux, spec: tt.fields.spec, pod: tt.fields.pod, podUID: tt.fields.podUID, diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 25358644e85..d739be18ee1 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -351,7 +351,7 @@ func (p *csiPlugin) RequiresRemount(spec *volume.Spec) bool { klog.V(5).Info(log("Failed to mark %q as republish required, err: %v", spec.Name(), err)) return false } - csiDriver, err := p.csiDriverLister.Get(driverName) + csiDriver, err := p.getCSIDriver(driverName) if err != nil { klog.V(5).Info(log("Failed to mark %q as republish required, err: %v", spec.Name(), err)) return false @@ -582,6 +582,20 @@ func (p *csiPlugin) SupportsBulkVolumeVerification() bool { } func (p *csiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + driver, err := GetCSIDriverName(spec) + if err != nil { + return false, err + } + csiDriver, err := p.getCSIDriver(driver) + if err != nil { + return false, err + } + if csiDriver.Spec.SELinuxMount != nil { + return *csiDriver.Spec.SELinuxMount, nil + } + return false, nil + } return false, nil } @@ -795,17 +809,7 @@ func (p *csiPlugin) ConstructBlockVolumeSpec(podUID types.UID, specVolName, mapP // skipAttach looks up CSIDriver object associated with driver name // to determine if driver requires attachment volume operation func (p *csiPlugin) skipAttach(driver string) (bool, error) { - kletHost, ok := p.host.(volume.KubeletVolumeHost) - if ok { - if err := kletHost.WaitForCacheSync(); err != nil { - return false, err - } - } - - if p.csiDriverLister == nil { - return false, errors.New("CSIDriver lister does not exist") - } - csiDriver, err := p.csiDriverLister.Get(driver) + csiDriver, err := p.getCSIDriver(driver) if err != nil { if apierrors.IsNotFound(err) { // Don't skip attach if CSIDriver does not exist @@ -819,6 +823,21 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) { return false, nil } +func (p *csiPlugin) getCSIDriver(driver string) (*storage.CSIDriver, error) { + kletHost, ok := p.host.(volume.KubeletVolumeHost) + if ok { + if err := kletHost.WaitForCacheSync(); err != nil { + return nil, err + } + } + + if p.csiDriverLister == nil { + return nil, errors.New("CSIDriver lister does not exist") + } + csiDriver, err := p.csiDriverLister.Get(driver) + return csiDriver, err +} + // supportsVolumeMode checks whether the CSI driver supports a volume in the given mode. // An error indicates that it isn't supported and explains why. func (p *csiPlugin) supportsVolumeLifecycleMode(driver string, volumeMode storage.VolumeLifecycleMode) error { @@ -836,14 +855,7 @@ func (p *csiPlugin) supportsVolumeLifecycleMode(driver string, volumeMode storag // optional), but then only persistent volumes are supported. var csiDriver *storage.CSIDriver if p.csiDriverLister != nil { - kletHost, ok := p.host.(volume.KubeletVolumeHost) - if ok { - if err := kletHost.WaitForCacheSync(); err != nil { - return err - } - } - - c, err := p.csiDriverLister.Get(driver) + c, err := p.getCSIDriver(driver) if err != nil && !apierrors.IsNotFound(err) { // Some internal error. return err @@ -904,14 +916,7 @@ func (p *csiPlugin) getFSGroupPolicy(driver string) (storage.FSGroupPolicy, erro // optional) var csiDriver *storage.CSIDriver if p.csiDriverLister != nil { - kletHost, ok := p.host.(volume.KubeletVolumeHost) - if ok { - if err := kletHost.WaitForCacheSync(); err != nil { - return storage.ReadWriteOnceWithFSTypeFSGroupPolicy, err - } - } - - c, err := p.csiDriverLister.Get(driver) + c, err := p.getCSIDriver(driver) if err != nil && !apierrors.IsNotFound(err) { // Some internal error. return storage.ReadWriteOnceWithFSTypeFSGroupPolicy, err @@ -969,16 +974,7 @@ func (p *csiPlugin) newAttacherDetacher() (*csiAttacher, error) { // podInfoEnabled check CSIDriver enabled pod info flag func (p *csiPlugin) podInfoEnabled(driverName string) (bool, error) { - kletHost, ok := p.host.(volume.KubeletVolumeHost) - if ok { - kletHost.WaitForCacheSync() - } - - if p.csiDriverLister == nil { - return false, fmt.Errorf("CSIDriverLister not found") - } - - csiDriver, err := p.csiDriverLister.Get(driverName) + csiDriver, err := p.getCSIDriver(driverName) if err != nil { if apierrors.IsNotFound(err) { klog.V(4).Infof(log("CSIDriver %q not found, not adding pod information", driverName)) From 49148ddfd00cf9768a5ddaf0564e587c5a6ac1b5 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 20 Jul 2022 13:22:40 +0200 Subject: [PATCH 12/22] Extract getSELinuxLabel from AddPodToVolume To keep the function smaller. --- .../cache/desired_state_of_world.go | 115 +++++++++--------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 3b7f6c35113..a5a47ad9e10 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -282,61 +282,9 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( volumeName = util.GetUniqueVolumeNameFromSpecWithPod(podName, volumePlugin, volumeSpec) } - var seLinuxFileLabel string - // Volume plugin supports SELinux context mount for all its volumes. - var pluginSupportsSELinuxContextMount bool - // The volume is ReadWriteOncePod. We don't support other volume types in SELinuxMountReadWriteOncePod feature. - // Don't use mount option to apply the SELinux context, still, track the context and report metrics of things - // that would break if the feature was for all volume access modes. - var isRWOP bool - - if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { - pluginSupportsSELinuxContextMount, err = dsw.getSELinuxMountSupport(volumeSpec) - if err != nil { - return "", err - } - isRWOP = util.IsRWOP(volumeSpec) - if pluginSupportsSELinuxContextMount { - // Ensure that a volume that can be mounted with "-o context=XYZ" is - // used only by containers with the same SELinux contexts. - for _, containerContext := range seLinuxContainerContexts { - newLabel, err := util.SELinuxOptionsToFileLabel(containerContext) - if err != nil { - fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) - if isRWOP { - // Cannot mount with -o context if the context can't be composed. - seLinuxContainerContextErrors.Add(1.0) - return "", fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - seLinuxContainerContextWarnings.Add(1.0) - klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") - break - } - } - if seLinuxFileLabel == "" { - seLinuxFileLabel = newLabel - continue - } - if seLinuxFileLabel != newLabel { - fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) - if isRWOP { - seLinuxPodContextMismatchErrors.Add(1.0) - return "", fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - seLinuxPodContextMismatchWarnings.Add(1.0) - klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") - break - } - } - } - } else { - // Volume plugin does not support SELinux context mount. - // DSW will track this volume with SELinux label "", i.e. no mount with - // -o context. - seLinuxFileLabel = "" - } + seLinuxFileLabel, pluginSupportsSELinuxContextMount, err := dsw.getSELinuxLabel(volumeSpec, seLinuxContainerContexts) + if err != nil { + return "", err } klog.V(4).InfoS("volume final SELinux label decided", "volume", volumeSpec.Name(), "label", seLinuxFileLabel) @@ -383,6 +331,7 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( if seLinuxFileLabel != vol.seLinuxFileLabel { // TODO: update the error message after tests, e.g. add at least the conflicting pod names. fullErr := fmt.Errorf("conflicting SELinux labels of volume %s: %q and %q", volumeSpec.Name(), vol.seLinuxFileLabel, seLinuxFileLabel) + isRWOP := util.IsRWOP(volumeSpec) if isRWOP { seLinuxVolumeContextMismatchErrors.Add(1.0) return "", fullErr @@ -418,6 +367,62 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( return volumeName, nil } +func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinuxContainerContexts []*v1.SELinuxOptions) (string, bool, error) { + var seLinuxFileLabel string + var pluginSupportsSELinuxContextMount bool + + if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + var err error + pluginSupportsSELinuxContextMount, err = dsw.getSELinuxMountSupport(volumeSpec) + if err != nil { + return "", false, err + } + isRWOP := util.IsRWOP(volumeSpec) + if pluginSupportsSELinuxContextMount { + // Ensure that a volume that can be mounted with "-o context=XYZ" is + // used only by containers with the same SELinux contexts. + for _, containerContext := range seLinuxContainerContexts { + newLabel, err := util.SELinuxOptionsToFileLabel(containerContext) + if err != nil { + fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) + if isRWOP { + // Cannot mount with -o context if the context can't be composed. + seLinuxContainerContextErrors.Add(1.0) + return "", false, fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + seLinuxContainerContextWarnings.Add(1.0) + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + break + } + } + if seLinuxFileLabel == "" { + seLinuxFileLabel = newLabel + continue + } + if seLinuxFileLabel != newLabel { + fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) + if isRWOP { + seLinuxPodContextMismatchErrors.Add(1.0) + return "", false, fullErr + } else { + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + seLinuxPodContextMismatchWarnings.Add(1.0) + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + break + } + } + } + } else { + // Volume plugin does not support SELinux context mount. + // DSW will track this volume with SELinux label "", i.e. no mount with + // -o context. + seLinuxFileLabel = "" + } + } + return seLinuxFileLabel, pluginSupportsSELinuxContextMount, nil +} + func (dsw *desiredStateOfWorld) MarkVolumesReportedInUse( reportedVolumes []v1.UniqueVolumeName) { dsw.Lock() From 4df3f5873731ce03d71c44b3ce36d26e87b20ecd Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 28 Jul 2022 14:47:22 +0200 Subject: [PATCH 13/22] Add SELinux feature check for iSCSI volume plugin In theory the check is not necessary, but for sake of robustness and completenes, let's check SELinuxMountReadWriteOncePod feature gate before assuming anything about SELinux labels. --- pkg/volume/iscsi/iscsi.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 7390e59b341..7f24a4e32ff 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -24,7 +24,9 @@ import ( "strconv" "strings" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" "k8s.io/utils/io" @@ -366,9 +368,12 @@ func (b *iscsiDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) e if err != nil { klog.Errorf("iscsi: failed to setup") } - // The volume must have been mounted in MountDevice with -o context. - // TODO: extract from mount table in GetAttributes() to be sure? - b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != "" + + if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + // The volume must have been mounted in MountDevice with -o context. + // TODO: extract from mount table in GetAttributes() to be sure? + b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != "" + } return err } From 8d6b721ddd342530230ff067d4e305816bcf9b26 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 28 Jul 2022 14:45:06 +0200 Subject: [PATCH 14/22] Extract SELinux context error handling into a common func Add handlerSELinuxMetricError() which bumps the right metric + either consumes a SELinux error or lets it propagate up the stack. --- .../cache/desired_state_of_world.go | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index a5a47ad9e10..28e9a79727b 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/metrics" "k8s.io/klog/v2" apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/features" @@ -332,13 +333,8 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( // TODO: update the error message after tests, e.g. add at least the conflicting pod names. fullErr := fmt.Errorf("conflicting SELinux labels of volume %s: %q and %q", volumeSpec.Name(), vol.seLinuxFileLabel, seLinuxFileLabel) isRWOP := util.IsRWOP(volumeSpec) - if isRWOP { - seLinuxVolumeContextMismatchErrors.Add(1.0) - return "", fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - seLinuxVolumeContextMismatchWarnings.Add(1.0) - klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxVolumeContextMismatchWarnings, seLinuxVolumeContextMismatchErrors); err != nil { + return "", err } } else { if seLinuxFileLabel != "" { @@ -385,15 +381,8 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux newLabel, err := util.SELinuxOptionsToFileLabel(containerContext) if err != nil { fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) - if isRWOP { - // Cannot mount with -o context if the context can't be composed. - seLinuxContainerContextErrors.Add(1.0) - return "", false, fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - seLinuxContainerContextWarnings.Add(1.0) - klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") - break + if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxContainerContextWarnings, seLinuxContainerContextErrors); err != nil { + return "", false, err } } if seLinuxFileLabel == "" { @@ -402,14 +391,8 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux } if seLinuxFileLabel != newLabel { fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) - if isRWOP { - seLinuxPodContextMismatchErrors.Add(1.0) - return "", false, fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - seLinuxPodContextMismatchWarnings.Add(1.0) - klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") - break + if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxPodContextMismatchWarnings, seLinuxPodContextMismatchErrors); err != nil { + return "", false, err } } } @@ -625,3 +608,17 @@ func (dsw *desiredStateOfWorld) MarkVolumeAttachability(volumeName v1.UniqueVolu func (dsw *desiredStateOfWorld) getSELinuxMountSupport(volumeSpec *volume.Spec) (bool, error) { return util.SupportsSELinuxContextMount(volumeSpec, dsw.volumePluginMgr) } + +// Based on isRWOP, bump the right warning / error metric and either consume the error or return it. +func handlerSELinuxMetricError(err error, isRWOP bool, warningMetric, errorMetric *metrics.Gauge) error { + if isRWOP { + // Cannot mount with -o context if the context can't be composed. + errorMetric.Add(1.0) + return err + } + + // This is not an error yet, but it will be when support for RWO and RWX volumes is added + warningMetric.Add(1.0) + klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") + return nil +} From d9f792633d64f8dfe90d689ea7ab297750b0292e Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 28 Jul 2022 16:01:05 +0200 Subject: [PATCH 15/22] Add AddPodToVolume unit tests with SELinux --- .../cache/desired_state_of_world.go | 4 + .../cache/desired_state_of_world_test.go | 371 +++++++++++++++++- pkg/volume/testing/testing.go | 3 +- 3 files changed, 375 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 28e9a79727b..d5e76ff3c54 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -304,6 +304,10 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( } } } + if !util.IsRWOP(volumeSpec) { + // Clear SELinux label for the volume with unsupported access modes. + seLinuxFileLabel = "" + } if seLinuxFileLabel != "" { seLinuxVolumesAdmitted.Add(1.0) } diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go index 593228ef42c..c79a8d071bd 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go @@ -19,10 +19,12 @@ package cache import ( "testing" - "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" @@ -597,6 +599,371 @@ func Test_AddPodToVolume_WithEmptyDirSizeLimit(t *testing.T) { verifyDesiredSizeLimitInVolumeDsw(t, pod2Name, pod2DesiredSizeLimitMap, dsw) } +// Calls AddPodToVolume() with a volume that support SELinux, but is ReadWriteMany. +// Verifies newly added pod/volume exists via PodExistsInVolume() without SELinux context +// VolumeExists() and GetVolumesToMount() and no errors. +func Test_AddPodToVolume_Positive_SELinuxNoRWOP(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + // Arrange + plugins := []volume.VolumePlugin{ + &volumetesting.FakeBasicVolumePlugin{ + Plugin: volumetesting.FakeVolumePlugin{ + PluginName: "basic", + SupportsSELinux: true, + }, + }, + } + volumePluginMgr := volume.VolumePluginMgr{} + fakeVolumeHost := volumetesting.NewFakeVolumeHost(t, + "", /* rootDir */ + nil, /* kubeClient */ + nil, /* plugins */ + ) + volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) + dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinux := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c1,c2", + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: &seLinux, + }, + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "myClaim", + }, + }, + }, + }, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basicPV", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, + }, + }, + } + podName := util.GetUniquePodName(pod) + seLinuxContainerContexts := []*v1.SELinuxOptions{&seLinux} + + // Act + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsDsw(t, generatedVolumeName, "" /* SELinux */, dsw) + verifyVolumeExistsInVolumesToMount( + t, generatedVolumeName, false /* expectReportedInUse */, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinux */, dsw) + verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) +} + +// Calls AddPodToVolume() with a volume that does not support SELinux. +// Verifies newly added pod/volume exists via PodExistsInVolume() without SELinux context +// VolumeExists() and GetVolumesToMount() and no errors. +func Test_AddPodToVolume_Positive_NoSELinuxPlugin(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + // Arrange + plugins := []volume.VolumePlugin{ + &volumetesting.FakeBasicVolumePlugin{ + Plugin: volumetesting.FakeVolumePlugin{ + PluginName: "basic", + SupportsSELinux: false, + }, + }, + } + volumePluginMgr := volume.VolumePluginMgr{} + fakeVolumeHost := volumetesting.NewFakeVolumeHost(t, + "", /* rootDir */ + nil, /* kubeClient */ + nil, /* plugins */ + ) + volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) + dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinux := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c1,c2", + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: &seLinux, + }, + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "myClaim", + }, + }, + }, + }, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basicPV", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + }, + }, + } + podName := util.GetUniquePodName(pod) + seLinuxContainerContexts := []*v1.SELinuxOptions{&seLinux} + + // Act + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsDsw(t, generatedVolumeName, "" /* SELinux */, dsw) + verifyVolumeExistsInVolumesToMount( + t, generatedVolumeName, false /* expectReportedInUse */, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "" /* SELinux */, dsw) + verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) +} + +// Calls AddPodToVolume() twice to add two pods with the same SELinuxContext +// to the same ReadWriteOncePod PV. +// Verifies newly added pod/volume exists via PodExistsInVolume() +// VolumeExists() and GetVolumesToMount() and no errors. +func Test_AddPodToVolume_Positive_ExistingPodSameSELinuxRWOP(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + // Arrange + plugins := []volume.VolumePlugin{ + &volumetesting.FakeBasicVolumePlugin{ + Plugin: volumetesting.FakeVolumePlugin{ + PluginName: "basic", + SupportsSELinux: true, + }, + }, + } + volumePluginMgr := volume.VolumePluginMgr{} + fakeVolumeHost := volumetesting.NewFakeVolumeHost(t, + "", /* rootDir */ + nil, /* kubeClient */ + nil, /* plugins */ + ) + volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) + dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinux := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c1,c2", + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: &seLinux, + }, + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "myClaim", + }, + }, + }, + }, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basicPV", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + }, + }, + } + podName := util.GetUniquePodName(pod) + seLinuxContainerContexts := []*v1.SELinuxOptions{&seLinux} + + // Act + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsDsw(t, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) + verifyVolumeExistsInVolumesToMount( + t, generatedVolumeName, false /* expectReportedInUse */, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) + verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) + + // Arrange: prepare a different pod with the same context + pod2 := pod.DeepCopy() + pod2.Name = "pod2" + pod2.UID = "pod2uid" + pod2Name := util.GetUniquePodName(pod) + + // Act + generatedVolumeName2, err := dsw.AddPodToVolume( + pod2Name, pod2, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts) + // Assert + if err != nil { + t.Fatalf("Second AddPodToVolume failed. Expected: Actual: <%v>", err) + } + if generatedVolumeName2 != generatedVolumeName { + t.Errorf("Expected second generatedVolumeName %s, got %s", generatedVolumeName, generatedVolumeName2) + } + + verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) +} + +// Calls AddPodToVolume() twice to add two pods with different SELinuxContext +// to the same ReadWriteOncePod PV. +// Verifies newly added pod/volume exists via PodExistsInVolume() +// VolumeExists() and GetVolumesToMount() and no errors. +func Test_AddPodToVolume_Negative_ExistingPodDifferentSELinuxRWOP(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + // Arrange + plugins := []volume.VolumePlugin{ + &volumetesting.FakeBasicVolumePlugin{ + Plugin: volumetesting.FakeVolumePlugin{ + PluginName: "basic", + SupportsSELinux: true, + }, + }, + } + volumePluginMgr := volume.VolumePluginMgr{} + fakeVolumeHost := volumetesting.NewFakeVolumeHost(t, + "", /* rootDir */ + nil, /* kubeClient */ + nil, /* plugins */ + ) + volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) + dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinux1 := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c1,c2", + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: &seLinux1, + }, + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "myClaim", + }, + }, + }, + }, + }, + } + + volumeSpec := &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basicPV", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + }, + }, + } + podName := util.GetUniquePodName(pod) + seLinuxContainerContexts := []*v1.SELinuxOptions{&seLinux1} + + // Act + generatedVolumeName, err := dsw.AddPodToVolume( + podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts) + + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsDsw(t, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) + verifyVolumeExistsInVolumesToMount( + t, generatedVolumeName, false /* expectReportedInUse */, dsw) + verifyPodExistsInVolumeDsw(t, podName, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) + verifyVolumeExistsWithSpecNameInVolumeDsw(t, podName, volumeSpec.Name(), dsw) + + // Arrange: prepare a different pod with the same context + seLinux2 := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c3,c4", + } + seLinuxContainerContexts2 := []*v1.SELinuxOptions{&seLinux2} + pod2 := pod.DeepCopy() + pod2.Name = "pod2" + pod2.UID = "pod2uid" + pod2.Spec.SecurityContext.SELinuxOptions = &seLinux2 + pod2Name := util.GetUniquePodName(pod) + + // Act + _, err = dsw.AddPodToVolume( + pod2Name, pod2, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, seLinuxContainerContexts2) + // Assert + if err == nil { + t.Fatalf("Second AddPodToVolume succeeded, expected a failure") + } + // Verify the original SELinux context is still in DSW + verifyPodExistsInVolumeDsw(t, pod2Name, generatedVolumeName, "system_u:object_r:container_file_t:s0:c1,c2", dsw) +} + func verifyVolumeExistsDsw( t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectedSELinuxContext string, dsw DesiredStateOfWorld) { volumeExists := dsw.VolumeExists(expectedVolumeName, expectedSELinuxContext) diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 65949ad148e..7e2f9cac554 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -181,6 +181,7 @@ type FakeVolumePlugin struct { LimitKey string ProvisionDelaySeconds int SupportsRemount bool + SupportsSELinux bool DisableNodeExpansion bool // default to false which means it is attachable by default @@ -285,7 +286,7 @@ func (plugin *FakeVolumePlugin) SupportsBulkVolumeVerification() bool { } func (plugin *FakeVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) { - return false, nil + return plugin.SupportsSELinux, nil } func (plugin *FakeVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { From 17d850ee0e945b4b7974103889939c58e6194d96 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 28 Jul 2022 17:32:01 +0200 Subject: [PATCH 16/22] Add interface for SELinuxOptionsToFileLabel github.com/opencontainers/selinux/go-selinux needs OS that supports SELinux and SELinux enabled in it to return useful data, therefore add an interface in front of it, so we can mock its behavior in unit tests. --- .../cache/desired_state_of_world.go | 18 +++- .../cache/desired_state_of_world_test.go | 30 ++++-- .../volumemanager/metrics/metrics_test.go | 3 +- .../desired_state_of_world_populator_test.go | 3 +- .../reconciler/reconciler_test.go | 48 ++++++---- pkg/kubelet/volumemanager/volume_manager.go | 3 +- pkg/volume/util/selinux.go | 92 +++++++++++++++++-- 7 files changed, 155 insertions(+), 42 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index d5e76ff3c54..37b0b96b44b 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -142,14 +142,15 @@ type VolumeToMount struct { } // NewDesiredStateOfWorld returns a new instance of DesiredStateOfWorld. -func NewDesiredStateOfWorld(volumePluginMgr *volume.VolumePluginMgr) DesiredStateOfWorld { +func NewDesiredStateOfWorld(volumePluginMgr *volume.VolumePluginMgr, seLinuxTranslator util.SELinuxLabelTranslator) DesiredStateOfWorld { if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { registerSELinuxMetrics() } return &desiredStateOfWorld{ - volumesToMount: make(map[v1.UniqueVolumeName]volumeToMount), - volumePluginMgr: volumePluginMgr, - podErrors: make(map[types.UniquePodName]sets.String), + volumesToMount: make(map[v1.UniqueVolumeName]volumeToMount), + volumePluginMgr: volumePluginMgr, + podErrors: make(map[types.UniquePodName]sets.String), + seLinuxTranslator: seLinuxTranslator, } } @@ -164,6 +165,8 @@ type desiredStateOfWorld struct { volumePluginMgr *volume.VolumePluginMgr // podErrors are errors caught by desiredStateOfWorldPopulator about volumes for a given pod. podErrors map[types.UniquePodName]sets.String + // seLinuxTranslator translates v1.SELinuxOptions to a file SELinux label. + seLinuxTranslator util.SELinuxLabelTranslator sync.RWMutex } @@ -373,6 +376,11 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { var err error + + if !dsw.seLinuxTranslator.SELinuxEnabled() { + return "", false, nil + } + pluginSupportsSELinuxContextMount, err = dsw.getSELinuxMountSupport(volumeSpec) if err != nil { return "", false, err @@ -382,7 +390,7 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux // Ensure that a volume that can be mounted with "-o context=XYZ" is // used only by containers with the same SELinux contexts. for _, containerContext := range seLinuxContainerContexts { - newLabel, err := util.SELinuxOptionsToFileLabel(containerContext) + newLabel, err := dsw.seLinuxTranslator.SELinuxOptionsToFileLabel(containerContext) if err != nil { fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxContainerContextWarnings, seLinuxContainerContextErrors); err != nil { diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go index c79a8d071bd..32dd140fbf6 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world_test.go @@ -37,7 +37,8 @@ import ( func Test_AddPodToVolume_Positive_NewPodNewVolume(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod3", @@ -82,7 +83,8 @@ func Test_AddPodToVolume_Positive_NewPodNewVolume(t *testing.T) { func Test_AddPodToVolume_Positive_ExistingPodExistingVolume(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod3", @@ -156,7 +158,8 @@ func Test_AddPodToVolume_Positive_NamesForDifferentPodsAndDifferentVolumes(t *te } volumePluginMgr := volume.VolumePluginMgr{} volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) - dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(&volumePluginMgr, seLinuxTranslator) testcases := map[string]struct { pod1 *v1.Pod @@ -289,7 +292,8 @@ func Test_AddPodToVolume_Positive_NamesForDifferentPodsAndDifferentVolumes(t *te func Test_DeletePodFromVolume_Positive_PodExistsVolumeExists(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod3", @@ -341,7 +345,8 @@ func Test_DeletePodFromVolume_Positive_PodExistsVolumeExists(t *testing.T) { func Test_MarkVolumesReportedInUse_Positive_NewPodNewVolume(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) pod1 := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -466,7 +471,8 @@ func Test_MarkVolumesReportedInUse_Positive_NewPodNewVolume(t *testing.T) { func Test_AddPodToVolume_WithEmptyDirSizeLimit(t *testing.T) { volumePluginMgr, _ := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) quantity1Gi := resource.MustParse("1Gi") quantity2Gi := resource.MustParse("2Gi") quantity3Gi := resource.MustParse("3Gi") @@ -621,7 +627,8 @@ func Test_AddPodToVolume_Positive_SELinuxNoRWOP(t *testing.T) { nil, /* plugins */ ) volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) - dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(&volumePluginMgr, seLinuxTranslator) seLinux := v1.SELinuxOptions{ User: "system_u", Role: "object_r", @@ -701,7 +708,8 @@ func Test_AddPodToVolume_Positive_NoSELinuxPlugin(t *testing.T) { nil, /* plugins */ ) volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) - dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(&volumePluginMgr, seLinuxTranslator) seLinux := v1.SELinuxOptions{ User: "system_u", Role: "object_r", @@ -782,7 +790,8 @@ func Test_AddPodToVolume_Positive_ExistingPodSameSELinuxRWOP(t *testing.T) { nil, /* plugins */ ) volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) - dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(&volumePluginMgr, seLinuxTranslator) seLinux := v1.SELinuxOptions{ User: "system_u", Role: "object_r", @@ -882,7 +891,8 @@ func Test_AddPodToVolume_Negative_ExistingPodDifferentSELinuxRWOP(t *testing.T) nil, /* plugins */ ) volumePluginMgr.InitPlugins(plugins, nil /* prober */, fakeVolumeHost) - dsw := NewDesiredStateOfWorld(&volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := NewDesiredStateOfWorld(&volumePluginMgr, seLinuxTranslator) seLinux1 := v1.SELinuxOptions{ User: "system_u", Role: "object_r", diff --git a/pkg/kubelet/volumemanager/metrics/metrics_test.go b/pkg/kubelet/volumemanager/metrics/metrics_test.go index cd30a8023c1..151967bb242 100644 --- a/pkg/kubelet/volumemanager/metrics/metrics_test.go +++ b/pkg/kubelet/volumemanager/metrics/metrics_test.go @@ -32,7 +32,8 @@ import ( func TestMetricCollection(t *testing.T) { volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(k8stypes.NodeName("node-name"), volumePluginMgr) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index 24769116004..5b9dd66ac4b 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -1308,7 +1308,8 @@ func createDswpWithVolumeWithCustomPluginMgr(t *testing.T, pv *v1.PersistentVolu fakePodManager := kubepod.NewBasicPodManager( podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager) - fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr, seLinuxTranslator) fakeASW := cache.NewActualStateOfWorld("fake", fakeVolumePluginMgr) fakeRuntime := &containertest.FakeRuntime{} fakeStateProvider := &fakePodStateProvider{} diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go index bd49d90b994..be7a26338f9 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler_test.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler_test.go @@ -70,7 +70,8 @@ func hasAddedPods() bool { return true } func Test_Run_Positive_DoNothing(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -114,7 +115,8 @@ func Test_Run_Positive_DoNothing(t *testing.T) { func Test_Run_Positive_VolumeAttachAndMount(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -204,7 +206,8 @@ func Test_Run_Positive_VolumeAttachAndMountMigrationEnabled(t *testing.T) { }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient(v1.AttachedVolume{ @@ -309,7 +312,8 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabled(t *testing.T) { }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -388,7 +392,8 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabled(t *testing.T) { func Test_Run_Negative_VolumeMountControllerAttachEnabled(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -466,7 +471,8 @@ func Test_Run_Negative_VolumeMountControllerAttachEnabled(t *testing.T) { func Test_Run_Positive_VolumeAttachMountUnmountDetach(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -568,7 +574,8 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) { }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -691,7 +698,8 @@ func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(gcepv, gcepvc) fakeRecorder := &record.FakeRecorder{} @@ -803,7 +811,8 @@ func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(gcepv, gcepvc, v1.AttachedVolume{ Name: "fake-plugin/fake-device1", @@ -903,7 +912,8 @@ func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(gcepv, gcepvc) fakeRecorder := &record.FakeRecorder{} @@ -1024,7 +1034,8 @@ func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(gcepv, gcepvc, v1.AttachedVolume{ Name: "fake-plugin/fake-device1", @@ -1293,7 +1304,8 @@ func Test_Run_Positive_VolumeFSResizeControllerAttachEnabled(t *testing.T) { }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(pv, pvc, v1.AttachedVolume{ Name: v1.UniqueVolumeName(fmt.Sprintf("fake-plugin/%s", tc.pvName)), @@ -1547,8 +1559,9 @@ func Test_UncertainDeviceGlobalMounts(t *testing.T) { } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) fakePlugin.SupportsRemount = tc.supportRemount + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(pv, pvc, v1.AttachedVolume{ Name: v1.UniqueVolumeName(fmt.Sprintf("fake-plugin/%s", tc.volumeName)), @@ -1770,7 +1783,8 @@ func Test_UncertainVolumeMountState(t *testing.T) { volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) fakePlugin.SupportsRemount = tc.supportRemount - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createtestClientWithPVPVC(pv, pvc, v1.AttachedVolume{ Name: v1.UniqueVolumeName(fmt.Sprintf("fake-plugin/%s", tc.volumeName)), @@ -2087,8 +2101,9 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabledRace(t *testing.T) { }, } volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNode(t, node) + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} @@ -2234,8 +2249,9 @@ func getReconciler(kubeletDir string, t *testing.T, volumePaths []string) (Recon node := getFakeNode() volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgrWithNodeAndRoot(t, node, kubeletDir) tmpKubeletPodDir := filepath.Join(kubeletDir, "pods") + seLinuxTranslator := util.NewFakeSELinuxLabelTranslator() - dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) + dsw := cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator) asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr) kubeClient := createTestClient() fakeRecorder := &record.FakeRecorder{} diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 0357ea0f5d1..ee58b2fcaa7 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -186,10 +186,11 @@ func NewVolumeManager( keepTerminatedPodVolumes bool, blockVolumePathHandler volumepathhandler.BlockVolumePathHandler) VolumeManager { + seLinuxTranslator := util.NewSELinuxLabelTranslator() vm := &volumeManager{ kubeClient: kubeClient, volumePluginMgr: volumePluginMgr, - desiredStateOfWorld: cache.NewDesiredStateOfWorld(volumePluginMgr), + desiredStateOfWorld: cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator), actualStateOfWorld: cache.NewActualStateOfWorld(nodeName, volumePluginMgr), operationExecutor: operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( kubeClient, diff --git a/pkg/volume/util/selinux.go b/pkg/volume/util/selinux.go index f05c30121f4..cd537610607 100644 --- a/pkg/volume/util/selinux.go +++ b/pkg/volume/util/selinux.go @@ -28,8 +28,36 @@ import ( "k8s.io/kubernetes/pkg/volume" ) -// SELinuxOptionsToFileLabel returns SELinux file label for given options. -func SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) { +// SELinuxLabelTranslator translates v1.SELinuxOptions of a process to SELinux file label. +type SELinuxLabelTranslator interface { + // SELinuxOptionsToFileLabel returns SELinux file label for given SELinuxOptions + // of a container process. + // When Role, User or Type are empty, they're read from the system defaults. + // It returns "" and no error on platforms that do not have SELinux enabled + // or don't support SELinux at all. + SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) + + // SELinuxEnabled returns true when the OS has enabled SELinux support. + SELinuxEnabled() bool +} + +// Real implementation of the interface. +// On Linux with SELinux enabled it translates. Otherwise it always returns an empty string and no error. +type translator struct{} + +var _ SELinuxLabelTranslator = &translator{} + +// NewSELinuxLabelTranslator returns new SELinuxLabelTranslator for the platform. +func NewSELinuxLabelTranslator() SELinuxLabelTranslator { + return &translator{} +} + +// SELinuxOptionsToFileLabel returns SELinux file label for given SELinuxOptions +// of a container process. +// When Role, User or Type are empty, they're read from the system defaults. +// It returns "" and no error on platforms that do not have SELinux enabled +// or don't support SELinux at all. +func (l *translator) SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) { if opts == nil { return "", nil } @@ -39,7 +67,6 @@ func SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) { return "", nil } - // TODO: use interface for InitLabels for unit tests. processLabel, fileLabel, err := label.InitLabels(args) if err != nil { // In theory, this should be unreachable. InitLabels can fail only when args contain an unknown option, @@ -75,13 +102,62 @@ func contextOptions(opts *v1.SELinuxOptions) []string { return args } -// SupportsSELinuxContextMount checks if the given volumeSpec supports with mount -o context -func SupportsSELinuxContextMount(volumeSpec *volume.Spec, volumePluginMgr *volume.VolumePluginMgr) (bool, error) { - // This is cheap - if !selinux.GetEnabled() { - return false, nil +func (l *translator) SELinuxEnabled() bool { + return selinux.GetEnabled() +} + +// Fake implementation of the interface for unit tests. +type fakeTranslator struct{} + +var _ SELinuxLabelTranslator = &fakeTranslator{} + +// NewFakeSELinuxLabelTranslator returns a fake translator for unit tests. +// It imitates a real translator on platforms that do not have SELinux enabled +// or don't support SELinux at all. +func NewFakeSELinuxLabelTranslator() SELinuxLabelTranslator { + return &fakeTranslator{} +} + +// SELinuxOptionsToFileLabel returns SELinux file label for given options. +func (l *fakeTranslator) SELinuxOptionsToFileLabel(opts *v1.SELinuxOptions) (string, error) { + if opts == nil { + return "", nil + } + // Fill empty values from "system defaults" (taken from Fedora Linux). + user := opts.User + if user == "" { + user = "system_u" } + role := opts.Role + if role == "" { + role = "object_r" + } + + // opts is context of the *process* to run in a container. Translate + // process type "container_t" to file label type "container_file_t". + // (The rest of the context is the same for processes and files). + fileType := opts.Type + if fileType == "" || fileType == "container_t" { + fileType = "container_file_t" + } + + level := opts.Level + if level == "" { + // If empty, level is allocated randomly. + level = "s0:c998,c999" + } + + ctx := fmt.Sprintf("%s:%s:%s:%s", user, role, fileType, level) + return ctx, nil +} + +func (l *fakeTranslator) SELinuxEnabled() bool { + return true +} + +// SupportsSELinuxContextMount checks if the given volumeSpec supports with mount -o context +func SupportsSELinuxContextMount(volumeSpec *volume.Spec, volumePluginMgr *volume.VolumePluginMgr) (bool, error) { plugin, _ := volumePluginMgr.FindPluginBySpec(volumeSpec) if plugin != nil { return plugin.SupportsSELinuxContextMount(volumeSpec) From 0793ecee3a3a4de997b4da002b2632c1432b4096 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 28 Jul 2022 19:30:44 +0200 Subject: [PATCH 17/22] Add unit tests for ASW.AddPodToVolume --- .../cache/actual_state_of_world.go | 3 +- .../cache/actual_state_of_world_test.go | 180 +++++++++++++++++- 2 files changed, 181 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index f20ae8be04b..241f122d883 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -1073,7 +1073,8 @@ func (asw *actualStateOfWorld) newAttachedVolume( DeviceMountPath: attachedVolume.deviceMountPath, PluginName: attachedVolume.pluginName, SELinuxMountContext: seLinuxMountContext}, - DeviceMountState: attachedVolume.deviceMountState, + DeviceMountState: attachedVolume.deviceMountState, + SELinuxMountContext: seLinuxMountContext, } } diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go index ecbb35fe55b..e68de416f51 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go @@ -18,13 +18,17 @@ package cache import ( "fmt" + "testing" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" - "testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" @@ -749,6 +753,137 @@ func Test_MarkDeviceAsMounted_Positive_NewVolume(t *testing.T) { verifyVolumeExistsInGloballyMountedVolumes(t, generatedVolumeName, asw) } +// Populates data struct with a volume with a SELinux context. +// Calls AddPodToVolume() to add a pod to the volume +// Verifies volume/pod combo exist using PodExistsInVolume() +func Test_AddPodToVolume_Positive_SELinux(t *testing.T) { + // Arrange + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + volumePluginMgr, plugin := volumetesting.GetTestKubeletVolumePluginMgr(t) + asw := NewActualStateOfWorld("mynode" /* nodeName */, volumePluginMgr) + devicePath := "fake/device/path" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "fake-device1", + }, + }, + }, + }, + }, + } + volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} + generatedVolumeName, err := util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec) + if err != nil { + t.Fatalf("GetUniqueVolumeNameFromSpec failed. Expected: Actual: <%v>", err) + } + + err = asw.MarkVolumeAsAttached(emptyVolumeName, volumeSpec, "" /* nodeName */, devicePath) + if err != nil { + t.Fatalf("MarkVolumeAsAttached failed. Expected: Actual: <%v>", err) + } + podName := util.GetUniquePodName(pod) + + mounter, err := plugin.NewMounter(volumeSpec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("NewMounter failed. Expected: Actual: <%v>", err) + } + + mapper, err := plugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{}) + if err != nil { + t.Fatalf("NewBlockVolumeMapper failed. Expected: Actual: <%v>", err) + } + + // Act + markVolumeOpts := operationexecutor.MarkVolumeOpts{ + PodName: podName, + PodUID: pod.UID, + VolumeName: generatedVolumeName, + Mounter: mounter, + BlockVolumeMapper: mapper, + OuterVolumeSpecName: volumeSpec.Name(), + VolumeSpec: volumeSpec, + SELinuxMountContext: "system_u:object_r:container_file_t:s0:c0,c1", + VolumeMountState: operationexecutor.VolumeMounted, + } + err = asw.AddPodToVolume(markVolumeOpts) + // Assert + if err != nil { + t.Fatalf("AddPodToVolume failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsAswWithSELinux(t, generatedVolumeName, "system_u:object_r:container_file_t:s0:c0,c1", asw) + verifyVolumeDoesntExistInUnmountedVolumes(t, generatedVolumeName, asw) + verifyVolumeDoesntExistInGloballyMountedVolumes(t, generatedVolumeName, asw) + verifyPodExistsInVolumeAsw(t, podName, generatedVolumeName, "fake/device/path" /* expectedDevicePath */, asw) + verifyVolumeExistsWithSpecNameInVolumeAsw(t, podName, volumeSpec.Name(), asw) + verifyVolumeMountedElsewhere(t, podName, generatedVolumeName, false /*expectedMountedElsewhere */, asw) +} + +// Calls MarkVolumeAsAttached() once to add volume +// Calls MarkDeviceAsMounted() with SELinux to mark volume as globally mounted. +// Verifies newly added volume exists in GetUnmountedVolumes() +// Verifies newly added volume exists in GetGloballyMountedVolumes() +func Test_MarkDeviceAsMounted_Positive_SELinux(t *testing.T) { + // Arrange + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + volumePluginMgr, plugin := volumetesting.GetTestKubeletVolumePluginMgr(t) + asw := NewActualStateOfWorld("mynode" /* nodeName */, volumePluginMgr) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + UID: "pod1uid", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "volume-name", + VolumeSource: v1.VolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "fake-device1", + }, + }, + }, + }, + }, + } + volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} + devicePath := "fake/device/path" + deviceMountPath := "fake/device/mount/path" + generatedVolumeName, err := util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec) + if err != nil { + t.Fatalf("GetUniqueVolumeNameFromSpec failed. Expected: Actual: <%v>", err) + } + + err = asw.MarkVolumeAsAttached(emptyVolumeName, volumeSpec, "" /* nodeName */, devicePath) + if err != nil { + t.Fatalf("MarkVolumeAsAttached failed. Expected: Actual: <%v>", err) + } + + // Act + err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath, deviceMountPath, "system_u:system_r:container_t:s0:c0,c1") + + // Assert + if err != nil { + t.Fatalf("MarkDeviceAsMounted failed. Expected: Actual: <%v>", err) + } + + verifyVolumeExistsAsw(t, generatedVolumeName, true /* shouldExist */, asw) + verifyVolumeExistsInUnmountedVolumes(t, generatedVolumeName, asw) + verifyVolumeExistsInGloballyMountedVolumesWithSELinux(t, generatedVolumeName, "system_u:system_r:container_t:s0:c0,c1", asw) +} + func TestUncertainVolumeMounts(t *testing.T) { // Arrange volumePluginMgr, plugin := volumetesting.GetTestKubeletVolumePluginMgr(t) @@ -849,6 +984,28 @@ func verifyVolumeExistsInGloballyMountedVolumes( globallyMountedVolumes) } +func verifyVolumeExistsInGloballyMountedVolumesWithSELinux( + t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectedSELinuxContext string, asw ActualStateOfWorld) { + globallyMountedVolumes := asw.GetGloballyMountedVolumes() + for _, volume := range globallyMountedVolumes { + if volume.VolumeName == expectedVolumeName { + if volume.SELinuxMountContext == expectedSELinuxContext { + return + } + t.Errorf( + "Volume %q has wrong SELinux context. Expected %q, got %q", + expectedVolumeName, + expectedSELinuxContext, + volume.SELinuxMountContext) + } + } + + t.Fatalf( + "Could not find volume %v in the list of GloballyMountedVolumes for actual state of world %+v", + expectedVolumeName, + globallyMountedVolumes) +} + func verifyVolumeDoesntExistInGloballyMountedVolumes( t *testing.T, volumeToCheck v1.UniqueVolumeName, asw ActualStateOfWorld) { globallyMountedVolumes := asw.GetGloballyMountedVolumes() @@ -876,6 +1033,27 @@ func verifyVolumeExistsAsw( } } +func verifyVolumeExistsAswWithSELinux( + t *testing.T, + expectedVolumeName v1.UniqueVolumeName, + expectedSELinuxContext string, + asw ActualStateOfWorld) { + volumes := asw.GetMountedVolumes() + for _, vol := range volumes { + if vol.VolumeName == expectedVolumeName { + if vol.SELinuxMountContext == expectedSELinuxContext { + return + } + t.Errorf( + "Volume %q has wrong SELinux context, expected %q, got %q", + expectedVolumeName, + expectedSELinuxContext, + vol.SELinuxMountContext) + } + } + t.Errorf("Volume %q not found in ASW", expectedVolumeName) +} + func verifyVolumeExistsInUnmountedVolumes( t *testing.T, expectedVolumeName v1.UniqueVolumeName, asw ActualStateOfWorld) { unmountedVolumes := asw.GetUnmountedVolumes() From 1490d51028b2330e332b5e3fb2f8539bfad323c4 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 29 Jul 2022 10:35:36 +0200 Subject: [PATCH 18/22] Remove noisy log The error would be logged every reconciler sync (100 ms). --- pkg/kubelet/volumemanager/cache/actual_state_of_world.go | 4 ---- pkg/kubelet/volumemanager/reconciler/reconciler.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 241f122d883..4b84c5f9f48 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -824,10 +824,6 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa fullErr := newSELinuxMountMismatchError(volumeName) if util.IsRWOP(volumeObj.spec) { return false, volumeObj.devicePath, fullErr - } else { - // This is not an error yet, but it will be when support for RWO and RWX volumes is added - // TODO: bump some metric here - klog.V(4).ErrorS(fullErr, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") } } } diff --git a/pkg/kubelet/volumemanager/reconciler/reconciler.go b/pkg/kubelet/volumemanager/reconciler/reconciler.go index 9ea0e961f40..35c14a54c8e 100644 --- a/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -221,7 +221,7 @@ func (rc *reconciler) mountOrAttachVolumes() { volumeToMount.DevicePath = devicePath if cache.IsSELinuxMountMismatchError(err) { // TODO: check error message + lower frequency, this can be noisy - klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("mount precondition failed", err).Error(), "pod", klog.KObj(volumeToMount.Pod)) + klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("mount precondition failed, please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file", err).Error(), "pod", klog.KObj(volumeToMount.Pod)) // TODO: report error better, this may be too noisy rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error()) } else if cache.IsVolumeNotAttachedError(err) { From a01e720a1a7def301cbd28cb7329ece5bf362114 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 29 Jul 2022 10:38:51 +0200 Subject: [PATCH 19/22] Rename IsRWOP To be able to update content of the function to other access modes when we implement SELinux mount for more of them. --- .../cache/actual_state_of_world.go | 2 +- .../cache/desired_state_of_world.go | 19 +++++++++---------- pkg/volume/util/selinux.go | 7 ++++++- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go index 4b84c5f9f48..e23dc4dad7d 100644 --- a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go @@ -822,7 +822,7 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa // The volume is mounted, check its SELinux context mount option if *volumeObj.seLinuxMountContext != seLinuxLabel { fullErr := newSELinuxMountMismatchError(volumeName) - if util.IsRWOP(volumeObj.spec) { + if util.VolumeSupportsSELinuxMount(volumeObj.spec) { return false, volumeObj.devicePath, fullErr } } diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index 37b0b96b44b..b8aad587047 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -307,7 +307,7 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( } } } - if !util.IsRWOP(volumeSpec) { + if !util.VolumeSupportsSELinuxMount(volumeSpec) { // Clear SELinux label for the volume with unsupported access modes. seLinuxFileLabel = "" } @@ -339,8 +339,8 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( if seLinuxFileLabel != vol.seLinuxFileLabel { // TODO: update the error message after tests, e.g. add at least the conflicting pod names. fullErr := fmt.Errorf("conflicting SELinux labels of volume %s: %q and %q", volumeSpec.Name(), vol.seLinuxFileLabel, seLinuxFileLabel) - isRWOP := util.IsRWOP(volumeSpec) - if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxVolumeContextMismatchWarnings, seLinuxVolumeContextMismatchErrors); err != nil { + supported := util.VolumeSupportsSELinuxMount(volumeSpec) + if err := handleSELinuxMetricError(fullErr, supported, seLinuxVolumeContextMismatchWarnings, seLinuxVolumeContextMismatchErrors); err != nil { return "", err } } else { @@ -385,7 +385,7 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux if err != nil { return "", false, err } - isRWOP := util.IsRWOP(volumeSpec) + seLinuxSupported := util.VolumeSupportsSELinuxMount(volumeSpec) if pluginSupportsSELinuxContextMount { // Ensure that a volume that can be mounted with "-o context=XYZ" is // used only by containers with the same SELinux contexts. @@ -393,7 +393,7 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux newLabel, err := dsw.seLinuxTranslator.SELinuxOptionsToFileLabel(containerContext) if err != nil { fullErr := fmt.Errorf("failed to construct SELinux label from context %q: %s", containerContext, err) - if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxContainerContextWarnings, seLinuxContainerContextErrors); err != nil { + if err := handleSELinuxMetricError(fullErr, seLinuxSupported, seLinuxContainerContextWarnings, seLinuxContainerContextErrors); err != nil { return "", false, err } } @@ -403,7 +403,7 @@ func (dsw *desiredStateOfWorld) getSELinuxLabel(volumeSpec *volume.Spec, seLinux } if seLinuxFileLabel != newLabel { fullErr := fmt.Errorf("volume %s is used with two different SELinux contexts in the same pod: %q, %q", volumeSpec.Name(), seLinuxFileLabel, newLabel) - if err := handlerSELinuxMetricError(fullErr, isRWOP, seLinuxPodContextMismatchWarnings, seLinuxPodContextMismatchErrors); err != nil { + if err := handleSELinuxMetricError(fullErr, seLinuxSupported, seLinuxPodContextMismatchWarnings, seLinuxPodContextMismatchErrors); err != nil { return "", false, err } } @@ -622,14 +622,13 @@ func (dsw *desiredStateOfWorld) getSELinuxMountSupport(volumeSpec *volume.Spec) } // Based on isRWOP, bump the right warning / error metric and either consume the error or return it. -func handlerSELinuxMetricError(err error, isRWOP bool, warningMetric, errorMetric *metrics.Gauge) error { - if isRWOP { - // Cannot mount with -o context if the context can't be composed. +func handleSELinuxMetricError(err error, seLinuxSupported bool, warningMetric, errorMetric *metrics.Gauge) error { + if seLinuxSupported { errorMetric.Add(1.0) return err } - // This is not an error yet, but it will be when support for RWO and RWX volumes is added + // This is not an error yet, but it will be when support for other access modes is added. warningMetric.Add(1.0) klog.V(4).ErrorS(err, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file") return nil diff --git a/pkg/volume/util/selinux.go b/pkg/volume/util/selinux.go index cd537610607..22854734f30 100644 --- a/pkg/volume/util/selinux.go +++ b/pkg/volume/util/selinux.go @@ -166,10 +166,15 @@ func SupportsSELinuxContextMount(volumeSpec *volume.Spec, volumePluginMgr *volum return false, nil } -func IsRWOP(volumeSpec *volume.Spec) bool { +// VolumeSupportsSELinuxMount returns true if given volume access mode can support mount with SELinux mount options. +func VolumeSupportsSELinuxMount(volumeSpec *volume.Spec) bool { + // Right now, SELinux mount is supported only for ReadWriteOncePod volumes. if !utilfeature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod) { return false } + if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + return false + } if volumeSpec.PersistentVolume == nil { return false } From 260912490e022ad95413b566d4264deaac5c838e Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 29 Jul 2022 10:48:59 +0200 Subject: [PATCH 20/22] Add a coment about handling same volumes with different contexts --- .../volumemanager/cache/desired_state_of_world.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index b8aad587047..31e9d621253 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -485,6 +485,19 @@ func (dsw *desiredStateOfWorld) VolumeExists( return false } if feature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) { + // Handling two volumes with the same name and different SELinux context + // as two *different* volumes here. Because if a volume is mounted with + // an old SELinux context, it must be unmounted first and then mounted again + // with the new context. + // + // This will happen when a pod A with context alpha_t runs and is being + // terminated by kubelet and its volumes are being torn down, while a + // pod B with context beta_t is already scheduled on the same node, + // using the same volumes + // The volumes from Pod A must be fully unmounted (incl. UnmountDevice) + // and mounted with new SELinux mount options for pod B. + // Without SELinux, kubelet can (and often does) reuse device mounted + // for A. return vol.seLinuxFileLabel == seLinuxMountContext } return true From 39f0d78714bba2bd18ec0fe3c302bc4326d0442d Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 29 Jul 2022 11:21:27 +0200 Subject: [PATCH 21/22] Add unit tests for GetPodVolumeNames --- pkg/volume/util/util_test.go | 119 +++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 5 deletions(-) diff --git a/pkg/volume/util/util_test.go b/pkg/volume/util/util_test.go index 0613eafc739..7df0fc10efd 100644 --- a/pkg/volume/util/util_test.go +++ b/pkg/volume/util/util_test.go @@ -578,11 +578,14 @@ func TestMakeAbsolutePath(t *testing.T) { } func TestGetPodVolumeNames(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() tests := []struct { - name string - pod *v1.Pod - expectedMounts sets.String - expectedDevices sets.String + name string + pod *v1.Pod + expectedMounts sets.String + expectedDevices sets.String + expectedSELinuxContexts map[string][]*v1.SELinuxOptions }{ { name: "empty pod", @@ -781,17 +784,123 @@ func TestGetPodVolumeNames(t *testing.T) { expectedMounts: sets.NewString("vol1", "vol2"), expectedDevices: sets.NewString(), }, + { + name: "pod with SELinuxOptions", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: &v1.SELinuxOptions{ + Type: "global_context_t", + Level: "s0:c1,c2", + }, + }, + InitContainers: []v1.Container{ + { + Name: "initContainer1", + SecurityContext: &v1.SecurityContext{ + SELinuxOptions: &v1.SELinuxOptions{ + Type: "initcontainer1_context_t", + Level: "s0:c3,c4", + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "vol1", + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: "container1", + SecurityContext: &v1.SecurityContext{ + SELinuxOptions: &v1.SELinuxOptions{ + Type: "container1_context_t", + Level: "s0:c5,c6", + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "vol1", + }, + { + Name: "vol2", + }, + }, + }, + { + Name: "container2", + // No SELinux context, will be inherited from PodSecurityContext + VolumeMounts: []v1.VolumeMount{ + { + Name: "vol2", + }, + { + Name: "vol3", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "vol1", + }, + { + Name: "vol2", + }, + { + Name: "vol3", + }, + }, + }, + }, + expectedMounts: sets.NewString("vol1", "vol2", "vol3"), + expectedSELinuxContexts: map[string][]*v1.SELinuxOptions{ + "vol1": { + { + Type: "initcontainer1_context_t", + Level: "s0:c3,c4", + }, + { + Type: "container1_context_t", + Level: "s0:c5,c6", + }, + }, + "vol2": { + { + Type: "container1_context_t", + Level: "s0:c5,c6", + }, + { + Type: "global_context_t", + Level: "s0:c1,c2", + }, + }, + "vol3": { + { + Type: "global_context_t", + Level: "s0:c1,c2", + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - mounts, devices, _ := GetPodVolumeNames(test.pod) + mounts, devices, contexts := GetPodVolumeNames(test.pod) if !mounts.Equal(test.expectedMounts) { t.Errorf("Expected mounts: %q, got %q", mounts.List(), test.expectedMounts.List()) } if !devices.Equal(test.expectedDevices) { t.Errorf("Expected devices: %q, got %q", devices.List(), test.expectedDevices.List()) } + if len(contexts) == 0 { + contexts = nil + } + if !reflect.DeepEqual(test.expectedSELinuxContexts, contexts) { + t.Errorf("Expected SELinuxContexts: %+v\ngot: %+v", test.expectedSELinuxContexts, contexts) + } }) } } From f9c7ce5b9c3912da3fa28ef1daf199320aaf7c08 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 29 Jul 2022 13:21:50 +0200 Subject: [PATCH 22/22] Add unit tests for DesiredStateOfWorldPopulator --- .../desired_state_of_world_populator_test.go | 193 +++++++++++++++++- pkg/volume/util/util_test.go | 3 + 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index 5b9dd66ac4b..aec172bf994 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -31,7 +31,9 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" + featuregatetesting "k8s.io/component-base/featuregate/testing" csitrans "k8s.io/csi-translation-lib" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/configmap" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" @@ -1067,6 +1069,195 @@ func TestCheckVolumeFSResize(t *testing.T) { } } +func TestCheckVolumeSELinux(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ReadWriteOncePod, true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)() + fullOpts := &v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c1,c2", + } + differentFullOpts := &v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_t", + Level: "s0:c9998,c9999", + } + partialOpts := &v1.SELinuxOptions{ + Level: "s0:c3,c4", + } + + testcases := []struct { + name string + accessModes []v1.PersistentVolumeAccessMode + existingContainerSELinuxOpts *v1.SELinuxOptions + newContainerSELinuxOpts *v1.SELinuxOptions + pluginSupportsSELinux bool + expectError bool + expectedContext string + }{ + { + name: "RWOP with plugin with SELinux with full context in pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + newContainerSELinuxOpts: fullOpts, + pluginSupportsSELinux: true, + expectedContext: "system_u:object_r:container_file_t:s0:c1,c2", + }, + { + name: "RWOP with plugin with SELinux with partial context in pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + newContainerSELinuxOpts: partialOpts, + pluginSupportsSELinux: true, + expectedContext: "system_u:object_r:container_file_t:s0:c3,c4", + }, + { + name: "RWX with plugin with SELinux with fill context in pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, + newContainerSELinuxOpts: fullOpts, + pluginSupportsSELinux: true, + expectedContext: "", // RWX volumes don't support SELinux + }, + { + name: "RWOP with plugin with no SELinux with fill context in pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + newContainerSELinuxOpts: fullOpts, + pluginSupportsSELinux: false, + expectedContext: "", // plugin doesn't support SELinux + }, + { + name: "RWOP with plugin with SELinux with no context in pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + newContainerSELinuxOpts: nil, + pluginSupportsSELinux: true, + expectedContext: "", + }, + { + name: "RWOP with plugin with SELinux with full context in pod with existing pod", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + existingContainerSELinuxOpts: fullOpts, + newContainerSELinuxOpts: fullOpts, + pluginSupportsSELinux: true, + expectedContext: "system_u:object_r:container_file_t:s0:c1,c2", + }, + { + name: "mismatched SELinux with RWX - success", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, + existingContainerSELinuxOpts: fullOpts, + newContainerSELinuxOpts: differentFullOpts, + pluginSupportsSELinux: true, + expectedContext: "", + }, + { + name: "mismatched SELinux with RWOP - failure", + accessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}, + existingContainerSELinuxOpts: fullOpts, + newContainerSELinuxOpts: differentFullOpts, + pluginSupportsSELinux: true, + expectError: true, + // The original seLinuxOpts are kept in DSW + expectedContext: "system_u:object_r:container_file_t:s0:c1,c2", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dswp-test-volume-name", + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{RBD: &v1.RBDPersistentVolumeSource{}}, + Capacity: volumeCapacity(1), + ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"}, + AccessModes: tc.accessModes, + }, + } + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: pv.Name, + Resources: v1.ResourceRequirements{ + Requests: pv.Spec.Capacity, + }, + AccessModes: tc.accessModes, + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + Capacity: pv.Spec.Capacity, + }, + } + + container := v1.Container{ + SecurityContext: &v1.SecurityContext{ + SELinuxOptions: nil, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: pv.Name, + MountPath: "/mnt", + }, + }, + } + + fakeVolumePluginMgr, plugin := volumetesting.GetTestKubeletVolumePluginMgr(t) + plugin.SupportsSELinux = tc.pluginSupportsSELinux + dswp, fakePodManager, fakeDSW, _, _ := createDswpWithVolumeWithCustomPluginMgr(t, pv, pvc, fakeVolumePluginMgr) + + var existingPod *v1.Pod + if tc.existingContainerSELinuxOpts != nil { + // Add existing pod + volume + existingContainer := container + existingContainer.SecurityContext.SELinuxOptions = tc.existingContainerSELinuxOpts + existingPod = createPodWithVolume("dswp-old-pod", "dswp-test-volume-name", "file-bound", []v1.Container{existingContainer}) + fakePodManager.AddPod(existingPod) + dswp.findAndAddNewPods() + } + + newContainer := container + newContainer.SecurityContext.SELinuxOptions = tc.newContainerSELinuxOpts + newPod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", []v1.Container{newContainer}) + + // Act - add the new Pod + fakePodManager.AddPod(newPod) + dswp.findAndAddNewPods() + + // Assert + + // Check the global volume state + uniquePodName := types.UniquePodName(newPod.UID) + uniqueVolumeName := v1.UniqueVolumeName("fake-plugin/" + newPod.Spec.Volumes[0].Name) + volumeExists := fakeDSW.VolumeExists(uniqueVolumeName, tc.expectedContext) + if !volumeExists { + t.Errorf( + "VolumeExists(%q) failed. Expected: Actual: <%v>", + uniqueVolumeName, + volumeExists) + } + + // Check the Pod local volume state + podExistsInVolume := fakeDSW.PodExistsInVolume(uniquePodName, uniqueVolumeName, tc.expectedContext) + if !podExistsInVolume && !tc.expectError { + t.Errorf( + "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", + podExistsInVolume) + } + if podExistsInVolume && tc.expectError { + t.Errorf( + "DSW PodExistsInVolume returned incorrect value. Expected: Actual: <%v>", + podExistsInVolume) + } + errors := fakeDSW.GetPodsWithErrors() + if tc.expectError && len(errors) == 0 { + t.Errorf("Expected Pod error, got none") + } + if !tc.expectError && len(errors) > 0 { + t.Errorf("Unexpected Pod errors: %v", errors) + } + verifyVolumeExistsInVolumesToMount(t, uniqueVolumeName, false /* expectReportedInUse */, fakeDSW) + }) + } +} + func createResizeRelatedVolumes(volumeMode *v1.PersistentVolumeMode) (pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) { pv = &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ @@ -1175,7 +1366,7 @@ func createPodWithVolume(pod, pv, pvc string, containers []v1.Container) *v1.Pod return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: pod, - UID: "dswp-test-pod-uid", + UID: kubetypes.UID(pod + "-uid"), Namespace: "dswp-test", }, Spec: v1.PodSpec{ diff --git a/pkg/volume/util/util_test.go b/pkg/volume/util/util_test.go index 7df0fc10efd..fe46963afbb 100644 --- a/pkg/volume/util/util_test.go +++ b/pkg/volume/util/util_test.go @@ -28,7 +28,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/slice" "k8s.io/kubernetes/pkg/volume" utilptr "k8s.io/utils/pointer"