From f2e23afcf13fef805ad5341bd7a5625de5982d6c Mon Sep 17 00:00:00 2001 From: Matthew Cary Date: Tue, 9 Jun 2020 21:30:40 +0000 Subject: [PATCH] Adds filtering of hosts to DialContexts. The provided DialContext wraps existing clients' DialContext in an attempt to preserve any existing timeout configuration. In some cases, we may replace infinite timeouts with golang defaults. - scaleio: tcp connect/keepalive values changed from 0/15 to 30/30 - storageos: no change --- api/api-rules/violation_exceptions.list | 3 + cmd/kube-controller-manager/app/core.go | 26 +++++++- cmd/kube-controller-manager/app/options/BUILD | 2 + .../app/options/options_test.go | 6 ++ .../persistentvolumebindercontroller.go | 20 ++++++ .../apis/config/zz_generated.deepcopy.go | 2 +- pkg/controller/volume/attachdetach/BUILD | 1 + .../attachdetach/attach_detach_controller.go | 36 ++++++---- .../attach_detach_controller_test.go | 5 +- pkg/controller/volume/expand/BUILD | 1 + .../volume/expand/expand_controller.go | 11 +++- .../volume/expand/expand_controller_test.go | 2 +- pkg/controller/volume/persistentvolume/BUILD | 1 + .../volume/persistentvolume/config/types.go | 6 ++ .../config/v1alpha1/defaults.go | 5 ++ .../v1alpha1/zz_generated.conversion.go | 10 +++ .../config/zz_generated.deepcopy.go | 5 ++ .../volume/persistentvolume/pv_controller.go | 4 ++ .../persistentvolume/pv_controller_base.go | 4 ++ .../volume/persistentvolume/volume_host.go | 5 ++ pkg/kubelet/BUILD | 1 + pkg/kubelet/volume_host.go | 6 ++ pkg/proxy/util/utils.go | 65 +++++++++++++++++++ pkg/proxy/util/utils_test.go | 33 ++++++++++ pkg/volume/BUILD | 1 + pkg/volume/plugins.go | 4 ++ pkg/volume/scaleio/BUILD | 1 + pkg/volume/scaleio/sio_client.go | 61 ++++++++++------- pkg/volume/scaleio/sio_mgr.go | 8 ++- pkg/volume/scaleio/sio_mgr_test.go | 7 +- pkg/volume/scaleio/sio_volume.go | 11 ++-- pkg/volume/storageos/BUILD | 1 + pkg/volume/storageos/storageos.go | 8 +-- pkg/volume/storageos/storageos_util.go | 11 +++- pkg/volume/storageos/storageos_util_test.go | 8 ++- pkg/volume/testing/BUILD | 1 + pkg/volume/testing/testing.go | 6 ++ .../config/v1alpha1/types.go | 6 ++ .../config/v1alpha1/zz_generated.deepcopy.go | 10 +++ staging/src/k8s.io/mount-utils/BUILD | 3 - test/integration/volume/attach_detach_test.go | 4 +- 41 files changed, 346 insertions(+), 65 deletions(-) diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 89a22b62a2c..d2fc81b01e9 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -370,6 +370,7 @@ API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1beta1,Po API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1beta1,PolicyRule,Verbs API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,GarbageCollectorControllerConfiguration,GCIgnoredResources API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Controllers +API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeBinderControllerConfiguration,VolumeHostCIDRDenylist API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyConfiguration,NodePortAddresses API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyIPVSConfiguration,ExcludeCIDRs API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,CAData @@ -569,6 +570,8 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,N API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,UnhealthyZoneThreshold API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeBinderControllerConfiguration,PVClaimBinderSyncPeriod API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeBinderControllerConfiguration,VolumeConfiguration +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeBinderControllerConfiguration,VolumeHostAllowLocalLoopback +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeBinderControllerConfiguration,VolumeHostCIDRDenylist API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeRecyclerConfiguration,IncrementTimeoutHostPath API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeRecyclerConfiguration,IncrementTimeoutNFS API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,PersistentVolumeRecyclerConfiguration,MaximumRetry diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 8d68f00621e..3826ac427a0 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -43,6 +43,7 @@ import ( servicecontroller "k8s.io/cloud-provider/controllers/service" "k8s.io/component-base/metrics/prometheus/ratelimiter" csitrans "k8s.io/csi-translation-lib" + "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" "k8s.io/kubernetes/pkg/controller" endpointcontroller "k8s.io/kubernetes/pkg/controller/endpoint" "k8s.io/kubernetes/pkg/controller/garbagecollector" @@ -285,6 +286,12 @@ func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, if err != nil { return nil, true, fmt.Errorf("failed to probe volume plugins when starting persistentvolume controller: %v", err) } + filteredDialOptions, err := options.ParseVolumeHostFilters( + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) + if err != nil { + return nil, true, err + } params := persistentvolumecontroller.ControllerParameters{ KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"), SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration, @@ -297,6 +304,7 @@ func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, PodInformer: ctx.InformerFactory.Core().V1().Pods(), NodeInformer: ctx.InformerFactory.Core().V1().Nodes(), EnableDynamicProvisioning: ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration.EnableDynamicProvisioning, + FilteredDialOptions: filteredDialOptions, } volumeController, volumeControllerErr := persistentvolumecontroller.NewController(params) if volumeControllerErr != nil { @@ -324,6 +332,13 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err return nil, true, fmt.Errorf("failed to probe volume plugins when starting attach/detach controller: %v", err) } + filteredDialOptions, err := options.ParseVolumeHostFilters( + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) + if err != nil { + return nil, true, err + } + attachDetachController, attachDetachControllerErr := attachdetach.NewAttachDetachController( ctx.ClientBuilder.ClientOrDie("attachdetach-controller"), @@ -340,6 +355,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync, ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration, attachdetach.DefaultTimerConfig, + filteredDialOptions, ) if attachDetachControllerErr != nil { return nil, true, fmt.Errorf("failed to start attach/detach controller: %v", attachDetachControllerErr) @@ -355,6 +371,12 @@ func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, err return nil, true, fmt.Errorf("failed to probe volume plugins when starting volume expand controller: %v", err) } csiTranslator := csitrans.New() + filteredDialOptions, err := options.ParseVolumeHostFilters( + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, + ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) + if err != nil { + return nil, true, err + } expandController, expandControllerErr := expand.NewExpandController( ctx.ClientBuilder.ClientOrDie("expand-controller"), ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), @@ -363,7 +385,9 @@ func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, err ctx.Cloud, plugins, csiTranslator, - csimigration.NewPluginManager(csiTranslator)) + csimigration.NewPluginManager(csiTranslator), + filteredDialOptions, + ) if expandControllerErr != nil { return nil, true, fmt.Errorf("failed to start volume expand controller: %v", expandControllerErr) diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index ba65d77052b..248338c19e9 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -63,6 +63,7 @@ go_library( "//pkg/controller/volume/attachdetach/config:go_default_library", "//pkg/controller/volume/persistentvolume/config:go_default_library", "//pkg/features:go_default_library", + "//pkg/proxy/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", @@ -78,6 +79,7 @@ go_library( "//staging/src/k8s.io/component-base/metrics:go_default_library", "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 59e0d252290..872415def16 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -100,6 +100,8 @@ var args = []string{ "--enable-taint-manager=false", "--cluster-signing-duration=10h", "--flex-volume-plugin-dir=/flex-volume-plugin", + "--volume-host-cidr-denylist=127.0.0.1/28,feed::/16", + "--volume-host-allow-local-loopback=false", "--horizontal-pod-autoscaler-downscale-delay=2m", "--horizontal-pod-autoscaler-sync-period=45s", "--horizontal-pod-autoscaler-upscale-delay=1m", @@ -350,6 +352,8 @@ func TestAddFlags(t *testing.T) { IncrementTimeoutHostPath: 45, }, }, + VolumeHostCIDRDenylist: []string{"127.0.0.1/28", "feed::/16"}, + VolumeHostAllowLocalLoopback: false, }, }, PodGCController: &PodGCControllerOptions{ @@ -589,6 +593,8 @@ func TestApplyTo(t *testing.T) { IncrementTimeoutHostPath: 45, }, }, + VolumeHostCIDRDenylist: []string{"127.0.0.1/28", "feed::/16"}, + VolumeHostAllowLocalLoopback: false, }, PodGCController: podgcconfig.PodGCControllerConfiguration{ TerminatedPodGCThreshold: 12000, diff --git a/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go b/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go index a09107706dd..4be5a29a9ca 100644 --- a/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go +++ b/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go @@ -17,9 +17,13 @@ limitations under the License. package options import ( + "fmt" + "github.com/spf13/pflag" persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" + netutils "k8s.io/utils/net" ) // PersistentVolumeBinderControllerOptions holds the PersistentVolumeBinderController options. @@ -43,6 +47,8 @@ func (o *PersistentVolumeBinderControllerOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", o.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features. HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.") fs.BoolVar(&o.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", o.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.") fs.StringVar(&o.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", o.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.") + fs.StringSliceVar(&o.VolumeHostCIDRDenylist, "volume-host-cidr-denylist", o.VolumeHostCIDRDenylist, "A comma-separated list of CIDR ranges to avoid from volume plugins.") + fs.BoolVar(&o.VolumeHostAllowLocalLoopback, "volume-host-allow-local-loopback", o.VolumeHostAllowLocalLoopback, "If false, deny local loopback IPs in addition to any CIDR ranges in --volume-host-cidr-denylist") } // ApplyTo fills up PersistentVolumeBinderController config with options. @@ -53,6 +59,8 @@ func (o *PersistentVolumeBinderControllerOptions) ApplyTo(cfg *persistentvolumec cfg.PVClaimBinderSyncPeriod = o.PVClaimBinderSyncPeriod cfg.VolumeConfiguration = o.VolumeConfiguration + cfg.VolumeHostCIDRDenylist = o.VolumeHostCIDRDenylist + cfg.VolumeHostAllowLocalLoopback = o.VolumeHostAllowLocalLoopback return nil } @@ -64,5 +72,17 @@ func (o *PersistentVolumeBinderControllerOptions) Validate() []error { } errs := []error{} + if _, err := ParseVolumeHostFilters(o.VolumeHostCIDRDenylist, o.VolumeHostAllowLocalLoopback); err != nil { + errs = append(errs, fmt.Errorf("Bad --volume-host-ip-denylist/--volume-host-allow-local-loopback %w", err)) + } return errs } + +// ParseVolumeHostFilters process the --volume-host-ip-denylist and --volume-host-allow-local-loopback flags. +func ParseVolumeHostFilters(denylist []string, allowLocalLoopback bool) (*proxyutil.FilteredDialOptions, error) { + denyCIDRs, err := netutils.ParseCIDRs(denylist) + if err != nil { + return nil, err + } + return &proxyutil.FilteredDialOptions{DialHostCIDRDenylist: denyCIDRs, AllowLocalLoopback: allowLocalLoopback}, nil +} diff --git a/pkg/controller/apis/config/zz_generated.deepcopy.go b/pkg/controller/apis/config/zz_generated.deepcopy.go index 0f5dd63275e..cba4ff2b8d0 100644 --- a/pkg/controller/apis/config/zz_generated.deepcopy.go +++ b/pkg/controller/apis/config/zz_generated.deepcopy.go @@ -123,7 +123,7 @@ func (in *KubeControllerManagerConfiguration) DeepCopyInto(out *KubeControllerMa out.NamespaceController = in.NamespaceController out.NodeIPAMController = in.NodeIPAMController out.NodeLifecycleController = in.NodeLifecycleController - out.PersistentVolumeBinderController = in.PersistentVolumeBinderController + in.PersistentVolumeBinderController.DeepCopyInto(&out.PersistentVolumeBinderController) out.PodGCController = in.PodGCController out.ReplicaSetController = in.ReplicaSetController out.ReplicationController = in.ReplicationController diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index cb412ec61f8..9e17ac460be 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/controller/volume/attachdetach/util:go_default_library", "//pkg/controller/volume/common:go_default_library", "//pkg/features:go_default_library", + "//pkg/proxy/util:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index dfcfe7b50d9..e5d703f5114 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -55,6 +55,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" "k8s.io/kubernetes/pkg/controller/volume/common" "k8s.io/kubernetes/pkg/features" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csimigration" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -117,21 +118,23 @@ func NewAttachDetachController( prober volume.DynamicPluginProber, disableReconciliationSync bool, reconcilerSyncDuration time.Duration, - timerConfig TimerConfig) (AttachDetachController, error) { + timerConfig TimerConfig, + filteredDialOptions *proxyutil.FilteredDialOptions) (AttachDetachController, error) { adc := &attachDetachController{ - kubeClient: kubeClient, - pvcLister: pvcInformer.Lister(), - pvcsSynced: pvcInformer.Informer().HasSynced, - pvLister: pvInformer.Lister(), - pvsSynced: pvInformer.Informer().HasSynced, - podLister: podInformer.Lister(), - podsSynced: podInformer.Informer().HasSynced, - podIndexer: podInformer.Informer().GetIndexer(), - nodeLister: nodeInformer.Lister(), - nodesSynced: nodeInformer.Informer().HasSynced, - cloud: cloud, - pvcQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"), + kubeClient: kubeClient, + pvcLister: pvcInformer.Lister(), + pvcsSynced: pvcInformer.Informer().HasSynced, + pvLister: pvInformer.Lister(), + pvsSynced: pvInformer.Informer().HasSynced, + podLister: podInformer.Lister(), + podsSynced: podInformer.Informer().HasSynced, + podIndexer: podInformer.Informer().GetIndexer(), + nodeLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + cloud: cloud, + pvcQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"), + filteredDialOptions: filteredDialOptions, } if utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && @@ -313,6 +316,9 @@ type attachDetachController struct { // intreeToCSITranslator translates from in-tree volume specs to CSI intreeToCSITranslator csimigration.InTreeToCSITranslator + + // filteredDialOptions configures any dialing done by the controller. + filteredDialOptions *proxyutil.FilteredDialOptions } func (adc *attachDetachController) Run(stopCh <-chan struct{}) { @@ -813,6 +819,10 @@ func (adc *attachDetachController) GetSubpather() subpath.Interface { return nil } +func (adc *attachDetachController) GetFilteredDialOptions() *proxyutil.FilteredDialOptions { + return adc.filteredDialOptions +} + func (adc *attachDetachController) GetCSIDriverLister() storagelistersv1.CSIDriverLister { return adc.csiDriverLister } diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 16988563438..a321478d9c5 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -55,6 +55,7 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { false, 5*time.Second, DefaultTimerConfig, + nil, /* filteredDialOptions */ ) // Assert @@ -175,7 +176,9 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 prober, false, 1*time.Second, - DefaultTimerConfig) + DefaultTimerConfig, + nil, /* filteredDialOptions */ + ) if err != nil { t.Fatalf("Run failed with error. Expected: Actual: <%v>", err) diff --git a/pkg/controller/volume/expand/BUILD b/pkg/controller/volume/expand/BUILD index b391fce2246..c4068adee66 100644 --- a/pkg/controller/volume/expand/BUILD +++ b/pkg/controller/volume/expand/BUILD @@ -9,6 +9,7 @@ go_library( deps = [ "//pkg/apis/core/v1/helper:go_default_library", "//pkg/controller/volume/events:go_default_library", + "//pkg/proxy/util:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", diff --git a/pkg/controller/volume/expand/expand_controller.go b/pkg/controller/volume/expand/expand_controller.go index 7d4e833a558..4fda7e15019 100644 --- a/pkg/controller/volume/expand/expand_controller.go +++ b/pkg/controller/volume/expand/expand_controller.go @@ -47,6 +47,7 @@ import ( cloudprovider "k8s.io/cloud-provider" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/controller/volume/events" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" @@ -104,6 +105,8 @@ type expandController struct { translator CSINameTranslator csiMigratedPluginManager csimigration.PluginManager + + filteredDialOptions *proxyutil.FilteredDialOptions } // NewExpandController expands the pvs @@ -115,7 +118,8 @@ func NewExpandController( cloud cloudprovider.Interface, plugins []volume.VolumePlugin, translator CSINameTranslator, - csiMigratedPluginManager csimigration.PluginManager) (ExpandController, error) { + csiMigratedPluginManager csimigration.PluginManager, + filteredDialOptions *proxyutil.FilteredDialOptions) (ExpandController, error) { expc := &expandController{ kubeClient: kubeClient, @@ -129,6 +133,7 @@ func NewExpandController( queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "volume_expand"), translator: translator, csiMigratedPluginManager: csiMigratedPluginManager, + filteredDialOptions: filteredDialOptions, } if err := expc.volumePluginMgr.InitPlugins(plugins, nil, expc); err != nil { @@ -449,3 +454,7 @@ func (expc *expandController) GetSubpather() subpath.Interface { // not needed for expand controller return nil } + +func (expc *expandController) GetFilteredDialOptions() *proxyutil.FilteredDialOptions { + return expc.filteredDialOptions +} diff --git a/pkg/controller/volume/expand/expand_controller_test.go b/pkg/controller/volume/expand/expand_controller_test.go index e0d0a52c7c4..df400aa0460 100644 --- a/pkg/controller/volume/expand/expand_controller_test.go +++ b/pkg/controller/volume/expand/expand_controller_test.go @@ -126,7 +126,7 @@ func TestSyncHandler(t *testing.T) { informerFactory.Storage().V1().StorageClasses().Informer().GetIndexer().Add(tc.storageClass) } translator := csitrans.New() - expc, err := NewExpandController(fakeKubeClient, pvcInformer, pvInformer, storageClassInformer, nil, allPlugins, translator, csimigration.NewPluginManager(translator)) + expc, err := NewExpandController(fakeKubeClient, pvcInformer, pvInformer, storageClassInformer, nil, allPlugins, translator, csimigration.NewPluginManager(translator), nil) if err != nil { t.Fatalf("error creating expand controller : %v", err) } diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD index 35584cc4406..dfc348bdbed 100644 --- a/pkg/controller/volume/persistentvolume/BUILD +++ b/pkg/controller/volume/persistentvolume/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/controller/volume/persistentvolume/metrics:go_default_library", "//pkg/controller/volume/persistentvolume/util:go_default_library", "//pkg/features:go_default_library", + "//pkg/proxy/util:go_default_library", "//pkg/util/goroutinemap:go_default_library", "//pkg/util/goroutinemap/exponentialbackoff:go_default_library", "//pkg/volume:go_default_library", diff --git a/pkg/controller/volume/persistentvolume/config/types.go b/pkg/controller/volume/persistentvolume/config/types.go index 11b28b0a53c..c840bd0b293 100644 --- a/pkg/controller/volume/persistentvolume/config/types.go +++ b/pkg/controller/volume/persistentvolume/config/types.go @@ -28,6 +28,12 @@ type PersistentVolumeBinderControllerConfiguration struct { PVClaimBinderSyncPeriod metav1.Duration // volumeConfiguration holds configuration for volume related features. VolumeConfiguration VolumeConfiguration + // VolumeHostCIDRDenylist is a list of CIDRs that should not be reachable by the + // controller from plugins. + VolumeHostCIDRDenylist []string + // VolumeHostAllowLocalLoopback indicates if local loopback hosts (127.0.0.1, etc) + // should be allowed from plugins. + VolumeHostAllowLocalLoopback bool } // VolumeConfiguration contains *all* enumerated flags meant to configure all volume diff --git a/pkg/controller/volume/persistentvolume/config/v1alpha1/defaults.go b/pkg/controller/volume/persistentvolume/config/v1alpha1/defaults.go index ce9481ac0cb..5fb6cbfcda5 100644 --- a/pkg/controller/volume/persistentvolume/config/v1alpha1/defaults.go +++ b/pkg/controller/volume/persistentvolume/config/v1alpha1/defaults.go @@ -39,6 +39,11 @@ func RecommendedDefaultPersistentVolumeBinderControllerConfiguration(obj *kubect obj.PVClaimBinderSyncPeriod = metav1.Duration{Duration: 15 * time.Second} } + if obj.VolumeHostAllowLocalLoopback == nil { + trueValue := true + obj.VolumeHostAllowLocalLoopback = &trueValue + } + // Use the default VolumeConfiguration options. RecommendedDefaultVolumeConfiguration(&obj.VolumeConfiguration) } diff --git a/pkg/controller/volume/persistentvolume/config/v1alpha1/zz_generated.conversion.go b/pkg/controller/volume/persistentvolume/config/v1alpha1/zz_generated.conversion.go index 910b6b0f5ce..e1cf08855b0 100644 --- a/pkg/controller/volume/persistentvolume/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/controller/volume/persistentvolume/config/v1alpha1/zz_generated.conversion.go @@ -21,6 +21,8 @@ limitations under the License. package v1alpha1 import ( + unsafe "unsafe" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" @@ -105,6 +107,10 @@ func autoConvert_v1alpha1_PersistentVolumeBinderControllerConfiguration_To_confi if err := Convert_v1alpha1_VolumeConfiguration_To_config_VolumeConfiguration(&in.VolumeConfiguration, &out.VolumeConfiguration, s); err != nil { return err } + out.VolumeHostCIDRDenylist = *(*[]string)(unsafe.Pointer(&in.VolumeHostCIDRDenylist)) + if err := v1.Convert_Pointer_bool_To_bool(&in.VolumeHostAllowLocalLoopback, &out.VolumeHostAllowLocalLoopback, s); err != nil { + return err + } return nil } @@ -113,6 +119,10 @@ func autoConvert_config_PersistentVolumeBinderControllerConfiguration_To_v1alpha if err := Convert_config_VolumeConfiguration_To_v1alpha1_VolumeConfiguration(&in.VolumeConfiguration, &out.VolumeConfiguration, s); err != nil { return err } + out.VolumeHostCIDRDenylist = *(*[]string)(unsafe.Pointer(&in.VolumeHostCIDRDenylist)) + if err := v1.Convert_bool_To_Pointer_bool(&in.VolumeHostAllowLocalLoopback, &out.VolumeHostAllowLocalLoopback, s); err != nil { + return err + } return nil } diff --git a/pkg/controller/volume/persistentvolume/config/zz_generated.deepcopy.go b/pkg/controller/volume/persistentvolume/config/zz_generated.deepcopy.go index 86c2e99e165..47ce776175b 100644 --- a/pkg/controller/volume/persistentvolume/config/zz_generated.deepcopy.go +++ b/pkg/controller/volume/persistentvolume/config/zz_generated.deepcopy.go @@ -25,6 +25,11 @@ func (in *PersistentVolumeBinderControllerConfiguration) DeepCopyInto(out *Persi *out = *in out.PVClaimBinderSyncPeriod = in.PVClaimBinderSyncPeriod out.VolumeConfiguration = in.VolumeConfiguration + if in.VolumeHostCIDRDenylist != nil { + in, out := &in.VolumeHostCIDRDenylist, &out.VolumeHostCIDRDenylist + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 61b1ed1f3ef..28ee91912d3 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -45,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/metrics" pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/features" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/goroutinemap" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" vol "k8s.io/kubernetes/pkg/volume" @@ -235,6 +236,9 @@ type PersistentVolumeController struct { translator CSINameTranslator csiMigratedPluginManager CSIMigratedPluginManager + + // filteredDialOptions configures any dialing done by the controller. + filteredDialOptions *proxyutil.FilteredDialOptions } // syncClaim is the main controller method to decide what to do with a claim. diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index 5747db35fd4..38e1d36e2bb 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -44,6 +44,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/common" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/metrics" pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/goroutinemap" vol "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csimigration" @@ -70,6 +71,7 @@ type ControllerParameters struct { NodeInformer coreinformers.NodeInformer EventRecorder record.EventRecorder EnableDynamicProvisioning bool + FilteredDialOptions *proxyutil.FilteredDialOptions } // NewController creates a new PersistentVolume controller @@ -142,6 +144,8 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) controller.translator = csiTranslator controller.csiMigratedPluginManager = csimigration.NewPluginManager(csiTranslator) + controller.filteredDialOptions = p.FilteredDialOptions + return controller, nil } diff --git a/pkg/controller/volume/persistentvolume/volume_host.go b/pkg/controller/volume/persistentvolume/volume_host.go index d75333f63d3..4030a844bcd 100644 --- a/pkg/controller/volume/persistentvolume/volume_host.go +++ b/pkg/controller/volume/persistentvolume/volume_host.go @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" vol "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/subpath" ) @@ -138,3 +139,7 @@ func (ctrl *PersistentVolumeController) GetSubpather() subpath.Interface { // No volume plugin needs Subpaths in PV controller. return nil } + +func (ctrl *PersistentVolumeController) GetFilteredDialOptions() *proxyutil.FilteredDialOptions { + return ctrl.filteredDialOptions +} diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 669b074c798..d13a5f03e55 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -98,6 +98,7 @@ go_library( "//pkg/kubelet/util/queue:go_default_library", "//pkg/kubelet/util/sliceutils:go_default_library", "//pkg/kubelet/volumemanager:go_default_library", + "//pkg/proxy/util:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/security/podsecuritypolicy/sysctl:go_default_library", "//pkg/util/iptables:go_default_library", diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index b9b6a0eae7b..c4e0361577d 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -38,6 +38,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/configmap" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/token" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" @@ -151,6 +152,11 @@ func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface { return kvh.kubelet.subpather } +func (kvh *kubeletVolumeHost) GetFilteredDialOptions() *proxyutil.FilteredDialOptions { + // FilteredDial is not needed in the kubelet. + return nil +} + func (kvh *kubeletVolumeHost) GetHostUtil() hostutil.HostUtils { return kvh.kubelet.hostutil } diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index c2bf2c97d15..4cf4e21eaba 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "net" + "net/http" "strconv" v1 "k8s.io/api/core/v1" @@ -124,6 +125,16 @@ func IsProxyableHostname(ctx context.Context, resolv Resolver, hostname string) return nil } +// IsAllowedHost checks if the given IP host address is in a network in the denied list. +func IsAllowedHost(host net.IP, denied []*net.IPNet) error { + for _, ipNet := range denied { + if ipNet.Contains(host) { + return ErrAddressNotAllowed + } + } + return nil +} + // GetLocalAddrs returns a list of all network addresses on the local system func GetLocalAddrs() ([]net.IP, error) { var localAddrs []net.IP @@ -311,3 +322,57 @@ func EnsureSysctl(sysctl utilsysctl.Interface, name string, newVal int) error { } return nil } + +// DialContext is a dial function matching the signature of net.Dialer.DialContext. +type DialContext = func(context.Context, string, string) (net.Conn, error) + +// FilteredDialOptions configures how a DialContext is wrapped by NewFilteredDialContext. +type FilteredDialOptions struct { + // DialHostIPDenylist restricts hosts from being dialed. + DialHostCIDRDenylist []*net.IPNet + // AllowLocalLoopback controls connections to local loopback hosts (as defined by + // IsProxyableIP). + AllowLocalLoopback bool +} + +// NewFilteredDialContext returns a DialContext function that filters connections based on a FilteredDialOptions. +func NewFilteredDialContext(wrapped DialContext, resolv Resolver, opts *FilteredDialOptions) DialContext { + if wrapped == nil { + wrapped = http.DefaultTransport.(*http.Transport).DialContext + } + if opts == nil { + // Do no filtering + return wrapped + } + if resolv == nil { + resolv = net.DefaultResolver + } + if len(opts.DialHostCIDRDenylist) == 0 && opts.AllowLocalLoopback { + // Do no filtering. + return wrapped + } + return func(ctx context.Context, network, address string) (net.Conn, error) { + resp, err := resolv.LookupIPAddr(ctx, address) + if err != nil { + return nil, err + } + + if len(resp) == 0 { + return nil, ErrNoAddresses + } + + for _, host := range resp { + if !opts.AllowLocalLoopback { + if err := isProxyableIP(host.IP); err != nil { + return nil, err + } + } + if opts.DialHostCIDRDenylist != nil { + if err := IsAllowedHost(host.IP, opts.DialHostCIDRDenylist); err != nil { + return nil, err + } + } + } + return wrapped(ctx, network, address) + } +} diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index e563a8da888..e7fa4dcea54 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -163,6 +163,39 @@ func TestIsProxyableHostname(t *testing.T) { } } +func TestIsAllowedHost(t *testing.T) { + testCases := []struct { + ip string + denied []string + want error + }{ + {"8.8.8.8", []string{}, nil}, + {"169.254.169.254", []string{"169.0.0.0/8"}, ErrAddressNotAllowed}, + {"169.254.169.254", []string{"fce8::/15", "169.254.169.0/24"}, ErrAddressNotAllowed}, + {"fce9:beef::", []string{"fce8::/15", "169.254.169.0/24"}, ErrAddressNotAllowed}, + {"127.0.0.1", []string{"127.0.0.1/32"}, ErrAddressNotAllowed}, + {"34.107.204.206", []string{"fce8::/15"}, nil}, + {"fce9:beef::", []string{"127.0.0.1/32"}, nil}, + {"34.107.204.206", []string{"127.0.0.1/32"}, nil}, + {"127.0.0.1", []string{}, nil}, + } + + for i := range testCases { + var denyList []*net.IPNet + for _, cidrStr := range testCases[i].denied { + _, ipNet, err := net.ParseCIDR(cidrStr) + if err != nil { + t.Fatalf("bad IP for test case: %v: %v", cidrStr, err) + } + denyList = append(denyList, ipNet) + } + got := IsAllowedHost(net.ParseIP(testCases[i].ip), denyList) + if testCases[i].want != got { + t.Errorf("case %d: expected %v, got %v", i, testCases[i].want, got) + } + } +} + func TestShouldSkipService(t *testing.T) { testCases := []struct { service *v1.Service diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD index 61c22c29bdd..c195f857c2a 100644 --- a/pkg/volume/BUILD +++ b/pkg/volume/BUILD @@ -18,6 +18,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/volume", visibility = ["//visibility:public"], deps = [ + "//pkg/proxy/util:go_default_library", "//pkg/volume/util/fs:go_default_library", "//pkg/volume/util/hostutil:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 9feccd04286..20edbb3ef60 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" "k8s.io/kubernetes/pkg/volume/util/subpath" @@ -450,6 +451,9 @@ type VolumeHost interface { // Returns an interface that should be used to execute subpath operations GetSubpather() subpath.Interface + + // Returns options to pass for proxyutil filtered dialers. + GetFilteredDialOptions() *proxyutil.FilteredDialOptions } // VolumePluginMgr tracks registered plugins. diff --git a/pkg/volume/scaleio/BUILD b/pkg/volume/scaleio/BUILD index d6aabc7d642..d5dd453e1f1 100644 --- a/pkg/volume/scaleio/BUILD +++ b/pkg/volume/scaleio/BUILD @@ -41,6 +41,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/scaleio", deps = [ + "//pkg/proxy/util:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/volume/scaleio/sio_client.go b/pkg/volume/scaleio/sio_client.go index cdd6c2db631..a9c83267194 100644 --- a/pkg/volume/scaleio/sio_client.go +++ b/pkg/volume/scaleio/sio_client.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" "os" "path/filepath" "regexp" @@ -33,6 +34,7 @@ import ( sio "github.com/thecodeteam/goscaleio" siotypes "github.com/thecodeteam/goscaleio/types/v1" "k8s.io/klog/v2" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" ) var ( @@ -56,37 +58,39 @@ type sioInterface interface { } type sioClient struct { - client *sio.Client - gateway string - username string - password string - insecure bool - certsEnabled bool - system *siotypes.System - sysName string - sysClient *sio.System - protectionDomain *siotypes.ProtectionDomain - pdName string - pdClient *sio.ProtectionDomain - storagePool *siotypes.StoragePool - spName string - spClient *sio.StoragePool - provisionMode string - sdcPath string - sdcGUID string - instanceID string - inited bool - diskRegex *regexp.Regexp - mtx sync.Mutex - exec utilexec.Interface + client *sio.Client + gateway string + username string + password string + insecure bool + certsEnabled bool + system *siotypes.System + sysName string + sysClient *sio.System + protectionDomain *siotypes.ProtectionDomain + pdName string + pdClient *sio.ProtectionDomain + storagePool *siotypes.StoragePool + spName string + spClient *sio.StoragePool + provisionMode string + sdcPath string + sdcGUID string + instanceID string + inited bool + diskRegex *regexp.Regexp + mtx sync.Mutex + exec utilexec.Interface + filteredDialOptions *proxyutil.FilteredDialOptions } -func newSioClient(gateway, username, password string, sslEnabled bool, exec utilexec.Interface) (*sioClient, error) { +func newSioClient(gateway, username, password string, sslEnabled bool, exec utilexec.Interface, filteredDialOptions *proxyutil.FilteredDialOptions) (*sioClient, error) { client := new(sioClient) client.gateway = gateway client.username = username client.password = password client.exec = exec + client.filteredDialOptions = filteredDialOptions if sslEnabled { client.insecure = false client.certsEnabled = true @@ -118,6 +122,15 @@ func (c *sioClient) init() error { klog.Error(log("failed to create client: %v", err)) return err } + transport, ok := client.Http.Transport.(*http.Transport) + if !ok { + return errors.New("could not set http.Transport options for scaleio client") + } + //lint:ignore SA1019 DialTLS must be used to support legacy clients. + if transport.DialTLS != nil { + return errors.New("DialTLS will be used instead of DialContext") + } + transport.DialContext = proxyutil.NewFilteredDialContext(transport.DialContext, nil, c.filteredDialOptions) c.client = client if _, err = c.client.Authenticate( &sio.ConfigConnect{ diff --git a/pkg/volume/scaleio/sio_mgr.go b/pkg/volume/scaleio/sio_mgr.go index d353fe58ff3..159f040e23a 100644 --- a/pkg/volume/scaleio/sio_mgr.go +++ b/pkg/volume/scaleio/sio_mgr.go @@ -21,6 +21,7 @@ import ( "strconv" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/volume" utilexec "k8s.io/utils/exec" siotypes "github.com/thecodeteam/goscaleio/types/v1" @@ -30,9 +31,10 @@ type sioMgr struct { client sioInterface configData map[string]string exec utilexec.Interface + host volume.VolumeHost } -func newSioMgr(configs map[string]string, exec utilexec.Interface) (*sioMgr, error) { +func newSioMgr(configs map[string]string, host volume.VolumeHost, exec utilexec.Interface) (*sioMgr, error) { if configs == nil { return nil, errors.New("missing configuration data") } @@ -41,7 +43,7 @@ func newSioMgr(configs map[string]string, exec utilexec.Interface) (*sioMgr, err configs[confKey.sdcRootPath] = defaultString(configs[confKey.sdcRootPath], sdcRootPath) configs[confKey.storageMode] = defaultString(configs[confKey.storageMode], "ThinProvisioned") - mgr := &sioMgr{configData: configs, exec: exec} + mgr := &sioMgr{configData: configs, host: host, exec: exec} return mgr, nil } @@ -61,7 +63,7 @@ func (m *sioMgr) getClient() (sioInterface, error) { certsEnabled := b klog.V(4).Info(log("creating new client for gateway %s", gateway)) - client, err := newSioClient(gateway, username, password, certsEnabled, m.exec) + client, err := newSioClient(gateway, username, password, certsEnabled, m.exec, m.host.GetFilteredDialOptions()) if err != nil { klog.Error(log("failed to create scaleio client: %v", err)) return nil, err diff --git a/pkg/volume/scaleio/sio_mgr_test.go b/pkg/volume/scaleio/sio_mgr_test.go index f818d2359b3..403b1300c7a 100644 --- a/pkg/volume/scaleio/sio_mgr_test.go +++ b/pkg/volume/scaleio/sio_mgr_test.go @@ -22,6 +22,7 @@ import ( "time" siotypes "github.com/thecodeteam/goscaleio/types/v1" + volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/utils/exec/testing" ) @@ -42,7 +43,8 @@ var ( ) func newTestMgr(t *testing.T) *sioMgr { - mgr, err := newSioMgr(fakeConfig, &testingexec.FakeExec{}) + host := volumetesting.NewFakeVolumeHost(t, "/tmp/fake", nil, nil) + mgr, err := newSioMgr(fakeConfig, host, &testingexec.FakeExec{}) if err != nil { t.Error(err) } @@ -51,7 +53,8 @@ func newTestMgr(t *testing.T) *sioMgr { } func TestMgrNew(t *testing.T) { - mgr, err := newSioMgr(fakeConfig, &testingexec.FakeExec{}) + host := volumetesting.NewFakeVolumeHost(t, "/tmp/fake", nil, nil) + mgr, err := newSioMgr(fakeConfig, host, &testingexec.FakeExec{}) if err != nil { t.Fatal(err) } diff --git a/pkg/volume/scaleio/sio_volume.go b/pkg/volume/scaleio/sio_volume.go index 337dba9cb0d..f6d6da975ac 100644 --- a/pkg/volume/scaleio/sio_volume.go +++ b/pkg/volume/scaleio/sio_volume.go @@ -405,7 +405,7 @@ func (v *sioVolume) setSioMgr() error { klog.Error(log("failed to retrieve sdc guid: %v", err)) return err } - mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) + mgr, err := newSioMgr(configData, v.plugin.host, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { klog.Error(log("failed to reset sio manager: %v", err)) @@ -444,8 +444,7 @@ func (v *sioVolume) resetSioMgr() error { klog.Error(log("failed to retrieve sdc guid: %v", err)) return err } - - mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) + mgr, err := newSioMgr(configData, v.plugin.host, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { klog.Error(log("failed to reset scaleio mgr: %v", err)) @@ -480,8 +479,7 @@ func (v *sioVolume) setSioMgrFromConfig() error { klog.Error(log("failed to load secret: %v", err)) return err } - - mgr, err := newSioMgr(data, v.plugin.host.GetExec(v.plugin.GetPluginName())) + mgr, err := newSioMgr(data, v.plugin.host, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { klog.Error(log("failed while setting scaleio mgr from config: %v", err)) @@ -516,8 +514,7 @@ func (v *sioVolume) setSioMgrFromSpec() error { klog.Error(log("failed to load secret: %v", err)) return err } - - mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) + mgr, err := newSioMgr(configData, v.plugin.host, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { klog.Error(log("failed to reset sio manager: %v", err)) diff --git a/pkg/volume/storageos/BUILD b/pkg/volume/storageos/BUILD index f66bb22b190..dffd6341a45 100644 --- a/pkg/volume/storageos/BUILD +++ b/pkg/volume/storageos/BUILD @@ -15,6 +15,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/storageos", deps = [ + "//pkg/proxy/util:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index 7243eaccc5c..e1ad8b9b8f3 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -110,7 +110,7 @@ func (plugin *storageosPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volu return nil, err } - return plugin.newMounterInternal(spec, pod, apiCfg, &storageosUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) + return plugin.newMounterInternal(spec, pod, apiCfg, &storageosUtil{host: plugin.host}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, apiCfg *storageosAPIConfig, manager storageosManager, mounter mount.Interface, exec utilexec.Interface) (volume.Mounter, error) { @@ -142,7 +142,7 @@ func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod } func (plugin *storageosPlugin) NewUnmounter(pvName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(pvName, podUID, &storageosUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) + return plugin.newUnmounterInternal(pvName, podUID, &storageosUtil{host: plugin.host}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } func (plugin *storageosPlugin) newUnmounterInternal(pvName string, podUID types.UID, manager storageosManager, mounter mount.Interface, exec utilexec.Interface) (volume.Unmounter, error) { @@ -194,7 +194,7 @@ func (plugin *storageosPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, er return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err) } - return plugin.newDeleterInternal(spec, apiCfg, &storageosUtil{}) + return plugin.newDeleterInternal(spec, apiCfg, &storageosUtil{host: plugin.host}) } func (plugin *storageosPlugin) newDeleterInternal(spec *volume.Spec, apiCfg *storageosAPIConfig, manager storageosManager) (volume.Deleter, error) { @@ -215,7 +215,7 @@ func (plugin *storageosPlugin) newDeleterInternal(spec *volume.Spec, apiCfg *sto } func (plugin *storageosPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { - return plugin.newProvisionerInternal(options, &storageosUtil{}) + return plugin.newProvisionerInternal(options, &storageosUtil{host: plugin.host}) } func (plugin *storageosPlugin) newProvisionerInternal(options volume.VolumeOptions, manager storageosManager) (volume.Provisioner, error) { diff --git a/pkg/volume/storageos/storageos_util.go b/pkg/volume/storageos/storageos_util.go index 00f2e09a684..8f080e39f9a 100644 --- a/pkg/volume/storageos/storageos_util.go +++ b/pkg/volume/storageos/storageos_util.go @@ -26,6 +26,8 @@ import ( storageosapi "github.com/storageos/go-api" storageostypes "github.com/storageos/go-api/types" "k8s.io/klog/v2" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" + "k8s.io/kubernetes/pkg/volume" utilexec "k8s.io/utils/exec" ) @@ -76,13 +78,17 @@ type apiImplementer interface { // storageosUtil is the utility structure to interact with the StorageOS API. type storageosUtil struct { - api apiImplementer + api apiImplementer + host volume.VolumeHost } func (u *storageosUtil) NewAPI(apiCfg *storageosAPIConfig) error { if u.api != nil { return nil } + if u.host == nil { + return errors.New("host must not be nil") + } if apiCfg == nil { apiCfg = &storageosAPIConfig{ apiAddr: defaultAPIAddress, @@ -98,6 +104,9 @@ func (u *storageosUtil) NewAPI(apiCfg *storageosAPIConfig) error { return err } api.SetAuth(apiCfg.apiUser, apiCfg.apiPass) + if err := api.SetDialContext(proxyutil.NewFilteredDialContext(api.GetDialContext(), nil, u.host.GetFilteredDialOptions())); err != nil { + return fmt.Errorf("failed to set DialContext in storageos client: %v", err) + } u.api = api return nil } diff --git a/pkg/volume/storageos/storageos_util_test.go b/pkg/volume/storageos/storageos_util_test.go index 2c00adce646..c658f71b044 100644 --- a/pkg/volume/storageos/storageos_util_test.go +++ b/pkg/volume/storageos/storageos_util_test.go @@ -49,8 +49,12 @@ func GetAPIConfig() *storageosAPIConfig { } func TestClient(t *testing.T) { - util := storageosUtil{} - err := util.NewAPI(GetAPIConfig()) + tmpDir, err := utiltesting.MkTmpdir("storageos_test") + if err != nil { + t.Fatalf("error creating tmpdir: %v", err) + } + util := storageosUtil{host: volumetest.NewFakeVolumeHost(t, tmpDir, nil, nil)} + err = util.NewAPI(GetAPIConfig()) if err != nil { t.Fatalf("error getting api config: %v", err) } diff --git a/pkg/volume/testing/BUILD b/pkg/volume/testing/BUILD index 58007e7f2e6..84e420944be 100644 --- a/pkg/volume/testing/BUILD +++ b/pkg/volume/testing/BUILD @@ -13,6 +13,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/testing", deps = [ + "//pkg/proxy/util:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/hostutil:go_default_library", diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index f19f913860b..f6884efaa64 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -46,6 +46,7 @@ import ( "k8s.io/client-go/tools/record" utiltesting "k8s.io/client-go/util/testing" cloudprovider "k8s.io/cloud-provider" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" . "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" @@ -118,6 +119,7 @@ type fakeVolumeHost struct { informerFactory informers.SharedInformerFactory kubeletErr error mux sync.Mutex + filteredDialOptions *proxyutil.FilteredDialOptions } var _ VolumeHost = &fakeVolumeHost{} @@ -207,6 +209,10 @@ func (f *fakeVolumeHost) GetSubpather() subpath.Interface { return f.subpather } +func (f *fakeVolumeHost) GetFilteredDialOptions() *proxyutil.FilteredDialOptions { + return f.filteredDialOptions +} + func (f *fakeVolumeHost) GetPluginMgr() *VolumePluginMgr { return f.pluginMgr } diff --git a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go index a92805d8c0f..c693f30d95a 100644 --- a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go @@ -471,6 +471,12 @@ type PersistentVolumeBinderControllerConfiguration struct { PVClaimBinderSyncPeriod metav1.Duration // volumeConfiguration holds configuration for volume related features. VolumeConfiguration VolumeConfiguration + // VolumeHostCIDRDenylist is a list of CIDRs that should not be reachable by the + // controller from plugins. + VolumeHostCIDRDenylist []string + // VolumeHostAllowLocalLoopback indicates if local loopback hosts (127.0.0.1, etc) + // should be allowed from plugins. + VolumeHostAllowLocalLoopback *bool } // PodGCControllerConfiguration contains elements describing PodGCController. diff --git a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go index 5e7fc064d95..453675db207 100644 --- a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go @@ -442,6 +442,16 @@ func (in *PersistentVolumeBinderControllerConfiguration) DeepCopyInto(out *Persi *out = *in out.PVClaimBinderSyncPeriod = in.PVClaimBinderSyncPeriod in.VolumeConfiguration.DeepCopyInto(&out.VolumeConfiguration) + if in.VolumeHostCIDRDenylist != nil { + in, out := &in.VolumeHostCIDRDenylist, &out.VolumeHostCIDRDenylist + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.VolumeHostAllowLocalLoopback != nil { + in, out := &in.VolumeHostAllowLocalLoopback, &out.VolumeHostAllowLocalLoopback + *out = new(bool) + **out = **in + } return } diff --git a/staging/src/k8s.io/mount-utils/BUILD b/staging/src/k8s.io/mount-utils/BUILD index d57b710fe82..26b12dc5555 100644 --- a/staging/src/k8s.io/mount-utils/BUILD +++ b/staging/src/k8s.io/mount-utils/BUILD @@ -35,9 +35,6 @@ go_library( "@io_bazel_rules_go//go/platform:freebsd": [ "//vendor/k8s.io/utils/io:go_default_library", ], - "@io_bazel_rules_go//go/platform:illumos": [ - "//vendor/k8s.io/utils/io:go_default_library", - ], "@io_bazel_rules_go//go/platform:ios": [ "//vendor/k8s.io/utils/io:go_default_library", ], diff --git a/test/integration/volume/attach_detach_test.go b/test/integration/volume/attach_detach_test.go index 4e6d0ffb9f4..116cdfc6b27 100644 --- a/test/integration/volume/attach_detach_test.go +++ b/test/integration/volume/attach_detach_test.go @@ -438,7 +438,9 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy nil, /* prober */ false, 5*time.Second, - timers) + timers, + nil, /* filteredDialOptions */ + ) if err != nil { t.Fatalf("Error creating AttachDetach : %v", err)