diff --git a/go.mod b/go.mod index 4937d4b7127..c242c498f95 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,6 @@ require ( github.com/google/go-cmp v0.3.0 github.com/google/gofuzz v1.0.0 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d - github.com/gophercloud/gophercloud v0.1.0 github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.7.0 // indirect github.com/hashicorp/golang-lru v0.5.1 @@ -91,7 +90,6 @@ require ( github.com/miekg/dns v1.1.4 github.com/mindprince/gonvml v0.0.0-20171110221305-fee913ce8fb2 // indirect github.com/mistifyio/go-zfs v2.1.1+incompatible // indirect - github.com/mitchellh/mapstructure v1.1.2 github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb // indirect github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/mrunalp/fileutils v0.0.0-20160930181131-4ee1cc9a8058 // indirect diff --git a/pkg/cloudprovider/providers/BUILD b/pkg/cloudprovider/providers/BUILD index ac4cc7fac12..6be8a179d17 100644 --- a/pkg/cloudprovider/providers/BUILD +++ b/pkg/cloudprovider/providers/BUILD @@ -16,10 +16,10 @@ go_library( "//cmd/kubelet/app:__pkg__", ], deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/vsphere:go_default_library", ], ) @@ -33,10 +33,7 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//pkg/cloudprovider/providers/openstack:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], ) diff --git a/pkg/cloudprovider/providers/openstack/doc.go b/pkg/cloudprovider/providers/openstack/doc.go deleted file mode 100644 index f970097aa5f..00000000000 --- a/pkg/cloudprovider/providers/openstack/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack diff --git a/pkg/cloudprovider/providers/providers.go b/pkg/cloudprovider/providers/providers.go index f258be3134f..8213c4802ca 100644 --- a/pkg/cloudprovider/providers/providers.go +++ b/pkg/cloudprovider/providers/providers.go @@ -20,9 +20,9 @@ package cloudprovider import ( // Cloud providers - _ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" _ "k8s.io/legacy-cloud-providers/aws" _ "k8s.io/legacy-cloud-providers/azure" _ "k8s.io/legacy-cloud-providers/gce" + _ "k8s.io/legacy-cloud-providers/openstack" _ "k8s.io/legacy-cloud-providers/vsphere" ) diff --git a/pkg/volume/cinder/BUILD b/pkg/volume/cinder/BUILD index e4c740756fb..a51e18c3d9f 100644 --- a/pkg/volume/cinder/BUILD +++ b/pkg/volume/cinder/BUILD @@ -17,7 +17,6 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/cinder", deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", @@ -33,6 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/keymutex:go_default_library", @@ -49,7 +49,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", @@ -60,6 +59,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 3d37dd86a09..040ea619789 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -25,18 +25,18 @@ import ( "path" "path/filepath" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" - "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/legacy-cloud-providers/openstack" "k8s.io/utils/keymutex" utilstrings "k8s.io/utils/strings" ) diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index ec1dbee7118..244a2cf8153 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -28,11 +28,11 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utiltesting "k8s.io/client-go/util/testing" - "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/legacy-cloud-providers/openstack" ) func TestCanSupport(t *testing.T) { diff --git a/staging/src/k8s.io/legacy-cloud-providers/BUILD b/staging/src/k8s.io/legacy-cloud-providers/BUILD index be967137ea3..65be89d13be 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/BUILD @@ -12,6 +12,7 @@ filegroup( "//staging/src/k8s.io/legacy-cloud-providers/aws:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/azure:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/gce:all-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/vsphere:all-srcs", ], tags = ["automanaged"], diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.mod b/staging/src/k8s.io/legacy-cloud-providers/go.mod index c81815964b9..94dcb406d93 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.mod +++ b/staging/src/k8s.io/legacy-cloud-providers/go.mod @@ -14,6 +14,8 @@ require ( github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534 github.com/aws/aws-sdk-go v1.16.26 github.com/dnaeon/go-vcr v1.0.1 // indirect + github.com/gophercloud/gophercloud v0.1.0 + github.com/mitchellh/mapstructure v1.1.2 github.com/prometheus/client_golang v0.9.2 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c github.com/satori/go.uuid v1.2.0 // indirect @@ -21,6 +23,7 @@ require ( github.com/vmware/govmomi v0.20.1 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f google.golang.org/api v0.6.1-0.20190607001116-5213b8090861 gopkg.in/gcfg.v1 v1.2.0 gopkg.in/warnings.v0 v0.1.1 // indirect diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.sum b/staging/src/k8s.io/legacy-cloud-providers/go.sum index 84054a4c976..f24dd8e12be 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.sum +++ b/staging/src/k8s.io/legacy-cloud-providers/go.sum @@ -100,6 +100,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -112,6 +113,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -135,6 +137,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -173,6 +177,7 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= diff --git a/pkg/cloudprovider/providers/openstack/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD similarity index 94% rename from pkg/cloudprovider/providers/openstack/BUILD rename to staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD index 6ce8090d2d3..ce61e815cc0 100644 --- a/pkg/cloudprovider/providers/openstack/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD @@ -9,7 +9,6 @@ load( go_library( name = "go_default_library", srcs = [ - "doc.go", "metadata.go", "openstack.go", "openstack_client.go", @@ -19,9 +18,9 @@ go_library( "openstack_routes.go", "openstack_volumes.go", ], - importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack", + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack", + importpath = "k8s.io/legacy-cloud-providers/openstack", deps = [ - "//pkg/util/mount:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -40,6 +39,7 @@ go_library( "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:go_default_library", "//vendor/github.com/gophercloud/gophercloud:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library", @@ -103,6 +103,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:all-srcs", + ], tags = ["automanaged"], ) diff --git a/pkg/cloudprovider/providers/openstack/MAINTAINERS.md b/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md similarity index 59% rename from pkg/cloudprovider/providers/openstack/MAINTAINERS.md rename to staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md index f71afec99fb..072e071c6b9 100644 --- a/pkg/cloudprovider/providers/openstack/MAINTAINERS.md +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md @@ -3,4 +3,4 @@ * [Angus Lees](https://github.com/anguslees) -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/pkg/cloudprovider/providers/openstack/MAINTAINERS.md?pixel)]() +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md?pixel)]() diff --git a/pkg/cloudprovider/providers/openstack/OWNERS b/staging/src/k8s.io/legacy-cloud-providers/openstack/OWNERS similarity index 100% rename from pkg/cloudprovider/providers/openstack/OWNERS rename to staging/src/k8s.io/legacy-cloud-providers/openstack/OWNERS diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go similarity index 98% rename from pkg/cloudprovider/providers/openstack/metadata.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go index f00e0e1c898..7562a684562 100644 --- a/pkg/cloudprovider/providers/openstack/metadata.go +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go @@ -30,7 +30,7 @@ import ( "strings" "k8s.io/klog" - "k8s.io/kubernetes/pkg/util/mount" + osmount "k8s.io/legacy-cloud-providers/openstack/util/mount" "k8s.io/utils/exec" ) @@ -125,7 +125,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { klog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) - mounter := mount.New("" /* default mount path */) + mounter := osmount.New("" /* default mount path */) err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"}) if err != nil { err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"}) diff --git a/pkg/cloudprovider/providers/openstack/metadata_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/metadata_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/metadata_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_client.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_client.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_client.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_client.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_instances.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_instances.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_instances.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_loadbalancer.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_loadbalancer.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_metrics.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_metrics.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_metrics.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_metrics.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_routes.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_routes_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_volumes.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_volumes.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_volumes.go diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD new file mode 100644 index 00000000000..79921a0102e --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "mount.go", + "mount_linux.go", + "mount_unsupported.go", + "mount_windows.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack/util/mount", + importpath = "k8s.io/legacy-cloud-providers/openstack/util/mount", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/io:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go new file mode 100644 index 00000000000..06a512ef412 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go @@ -0,0 +1,380 @@ +// +build !providerless + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mount provides a linux-centric library for mounting filesystems/ +// Vendored for the OpenStack provider and will be removed in 1.17 +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// FileType is one of the types of supported Linux file types. +type FileType string + +// A complete list of the support file types in the mount package. +const ( + // Default mount command if mounter path is not specified + defaultMountCommand = "mount" + MountsInGlobalPDPath = "mounts" + FileTypeDirectory FileType = "Directory" + FileTypeFile FileType = "File" + FileTypeSocket FileType = "Socket" + FileTypeCharDev FileType = "CharDevice" + FileTypeBlockDev FileType = "BlockDevice" +) + +// Interface is the general interface to filesystem mounts. +type Interface interface { + // Mount mounts source to target as fstype with given options. + Mount(source string, target string, fstype string, options []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts is not guaranteed consistent (i.e. + // it could change between chunked reads). This is guaranteed to be + // consistent. + List() ([]MntPoint, error) + // IsMountPointMatch determines if the mountpoint matches the dir + IsMountPointMatch(mp MntPoint, dir string) bool + // IsNotMountPoint determines if a directory is a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsNotMountPoint is more expensive than IsLikelyNotMountPoint. + // IsNotMountPoint detects bind mounts in linux. + // IsNotMountPoint enumerates all the mountpoints using List() and + // the list of mountpoints may be large, then it uses + // IsMountPointMatch to evaluate whether the directory is a mountpoint + IsNotMountPoint(file string) (bool, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts. + IsLikelyNotMountPoint(file string) (bool, error) + // DeviceOpened determines if the device is in use elsewhere + // on the system, i.e. still mounted. + DeviceOpened(pathname string) (bool, error) + // PathIsDevice determines if a path is a device. + PathIsDevice(pathname string) (bool, error) + // GetDeviceNameFromMount finds the device name by checking the mount path + // to get the global mount path which matches its plugin directory + GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) + // MakeRShared checks that given path is on a mount with 'rshared' mount + // propagation. If not, it bind-mounts the path as rshared. + MakeRShared(path string) error + // GetFileType checks for file/directory/socket/block/character devices. + // Will operate in the host mount namespace if kubelet is running in a container + GetFileType(pathname string) (FileType, error) + // MakeFile creates an empty file. + // Will operate in the host mount namespace if kubelet is running in a container + MakeFile(pathname string) error + // MakeDir creates a new directory. + // Will operate in the host mount namespace if kubelet is running in a container + MakeDir(pathname string) error + // SafeMakeDir creates subdir within given base. It makes sure that the + // created directory does not escape given base directory mis-using + // symlinks. Note that the function makes sure that it creates the directory + // somewhere under the base, nothing else. E.g. if the directory already + // exists, it may exist outside of the base due to symlinks. + // This method should be used if the directory to create is inside volume + // that's under user control. User must not be able to use symlinks to + // escape the volume to create directories somewhere else. + SafeMakeDir(subdir string, base string, perm os.FileMode) error + // Will operate in the host mount namespace if kubelet is running in a container. + // Error is returned on any other error than "file not found". + ExistsPath(pathname string) (bool, error) + // EvalHostSymlinks returns the path name after evaluating symlinks. + // Will operate in the host mount namespace if kubelet is running in a container. + EvalHostSymlinks(pathname string) (string, error) + // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given + // pod volume directory. + CleanSubPaths(podDir string, volumeName string) error + // PrepareSafeSubpath does everything that's necessary to prepare a subPath + // that's 1) inside given volumePath and 2) immutable after this call. + // + // newHostPath - location of prepared subPath. It should be used instead of + // hostName when running the container. + // cleanupAction - action to run when the container is running or it failed to start. + // + // CleanupAction must be called immediately after the container with given + // subpath starts. On the other hand, Interface.CleanSubPaths must be called + // when the pod finishes. + PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) + // GetMountRefs finds all mount references to the path, returns a + // list of paths. Path could be a mountpoint path, device or a normal + // directory (for bind mount). + GetMountRefs(pathname string) ([]string, error) + // GetFSGroup returns FSGroup of the path. + GetFSGroup(pathname string) (int64, error) + // GetSELinuxSupport returns true if given path is on a mount that supports + // SELinux. + GetSELinuxSupport(pathname string) (bool, error) + // GetMode returns permissions of the path. + GetMode(pathname string) (os.FileMode, error) +} + +// Subpath is the string path of the volume mount. +type Subpath struct { + // index of the VolumeMount for this container + VolumeMountIndex int + // Full path to the subpath directory on the host + Path string + // name of the volume that is a valid directory name. + VolumeName string + // Full path to the volume path + VolumePath string + // Path to the pod's directory, including pod UID + PodDir string + // Name of the container + ContainerName string +} + +// Exec executes command where mount utilities are. This can be either the host, +// container where kubelet runs or even a remote pod with mount utilities. +// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does +// not provide stdin/stdout/stderr streams. +type Exec interface { + // Run executes a command and returns its stdout + stderr combined in one + // stream. + Run(cmd string, args ...string) ([]byte, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface +var _ Interface = &Mounter{} + +// MntPoint represents a single line in /proc/mounts or /etc/fstab. +type MntPoint struct { + Device string + Path string + Type string + Opts []string + Freq int + Pass int +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.formatAndMount(source, target, fstype, options) +} + +// GetDeviceNameFromMount when given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code. +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// This method uses the List() of all mountpoints +// It is more extensive than IsLikelyNotMountPoint +// and it detects bind mounts in linux +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + // IsLikelyNotMountPoint provides a quick check + // to determine whether file IS A mountpoint + notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil { + return notMnt, notMntErr + } + // identified as mountpoint, so return this fact + if notMnt == false { + return notMnt, nil + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts + resolvedFile, err := mounter.EvalHostSymlinks(file) + if err != nil { + return true, err + } + + // check all mountpoints since IsLikelyNotMountPoint + // is not reliable for some mountpoint types + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return notMnt, mountPointsErr + } + for _, mp := range mountPoints { + if mounter.IsMountPointMatch(mp, resolvedFile) { + notMnt = false + break + } + } + return notMnt, nil +} + +// isBind detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func isBind(options []string) (bool, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + case "remount": + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + return bind, bindOpts, bindRemountOpts +} + +func checkForNetDev(options []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + return false +} + +// HasMountRefs checks the mountPath contains any of the mountRefs. +// This is a workaround for the unmount device issue caused by gci mounter. +// In GCI cluster, if gci mounter is used for mounting, the container started by mounter +// script will cause additional mounts created in the container. Since these mounts are +// irrelevant to the original mounts, they should be not considered when checking the +// mount references. Current solution is to filter out those mount paths that contain +// the string of original mount path. +// Plan to work on better approach to solve this issue. +func HasMountRefs(mountPath string, mountRefs []string) bool { + count := 0 + for _, ref := range mountRefs { + if !strings.Contains(ref, mountPath) { + count = count + 1 + } + } + return count > 0 +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if startsWithBackstep(rel) { + // Needed to escape the base path + return false + } + return true +} + +// startsWithBackstep checks if the given path starts with a backstep segment +func startsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} + +// getFileType checks for file/directory/socket and block/character devices +func getFileType(pathname string) (FileType, error) { + var pathType FileType + info, err := os.Stat(pathname) + if os.IsNotExist(err) { + return pathType, fmt.Errorf("path %q does not exist", pathname) + } + // err in call to os.Stat + if err != nil { + return pathType, err + } + + // checks whether the mode is the target mode + isSpecificMode := func(mode, targetMode os.FileMode) bool { + return mode&targetMode == targetMode + } + + mode := info.Mode() + if mode.IsDir() { + return FileTypeDirectory, nil + } else if mode.IsRegular() { + return FileTypeFile, nil + } else if isSpecificMode(mode, os.ModeSocket) { + return FileTypeSocket, nil + } else if isSpecificMode(mode, os.ModeDevice) { + if isSpecificMode(mode, os.ModeCharDevice) { + return FileTypeCharDev, nil + } + return FileTypeBlockDev, nil + } + + return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go new file mode 100644 index 00000000000..5dda471c663 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go @@ -0,0 +1,1345 @@ +// +build linux,!providerless + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + utilexec "k8s.io/utils/exec" + utilio "k8s.io/utils/io" + utilpath "k8s.io/utils/path" +) + +const ( + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 3 + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 + + // place for subpath mounts + containerSubPathDirectoryName = "volume-subpaths" + // syscall.Openat flags used to traverse directories not following symlinks + nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW + // flags for getting file descriptor without following the symlink + openFDFlags = unix.O_NOFOLLOW | unix.O_PATH +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts := isBind(options) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs") + if fsTypesNeedMounter.Has(fstype) { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { + mountArgs := makeMountArgs(source, target, fstype, options) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountCmd = mounterPath + } + + if mounter.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs = addSystemdScope("systemd-run", target, mountCmd, mountArgs) + } //else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + //} + + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + args := strings.Join(mountArgs, " ") + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, args, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Info("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Info("Detected OS with systemd") + return true +} + +// makeMountArgs makes the arguments to the mount(8) command. +func makeMountArgs(source, target, fstype string, options []string) []string { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs := []string{} + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + } + if len(options) > 0 { + mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + } + mountArgs = append(mountArgs, target) + + return mountArgs +} + +// addSystemdScope adds "system-run --scope" to given command line +func addSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MntPoint, error) { + return listProcMounts(procMountsPath) +} + +// IsMountPointMatch determines if a directory is a mountpoint. +func (mounter *Mounter) IsMountPointMatch(mp MntPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return ((mp.Path == dir) || (mp.Path == deletedDir)) +} + +// IsNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// if of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. +// If pathname is not a device, log and return false with nil error. +// If open returns errno EBUSY, return true with nil error. +// If open returns nil, return false with nil error. +// Otherwise, return false with error +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return exclusiveOpenFailsOnDevice(pathname) +} + +// PathIsDevice uses FileInfo returned from os.Stat to check if path refers +// to a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + pathType, err := mounter.GetFileType(pathname) + isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev + return isDevice, err +} + +func exclusiveOpenFailsOnDevice(pathname string) (bool, error) { + var isDevice bool + finfo, err := os.Stat(pathname) + if os.IsNotExist(err) { + isDevice = false + } + // err in call to os.Stat + if err != nil { + return false, fmt.Errorf( + "PathIsDevice failed for path %q: %v", + pathname, + err) + } + // path refers to a device + if finfo.Mode()&os.ModeDevice != 0 { + isDevice = true + } + + if !isDevice { + klog.Errorf("Path %q is not referring to a device.", pathname) + return false, nil + } + fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0) + // If the device is in use, open will return an invalid fd. + // When this happens, it is expected that Close will fail and throw an error. + defer unix.Close(fd) + if errno == nil { + // device not in use + return false, nil + } else if errno == unix.EBUSY { + // device is in use + return true, nil + } + // error during call to Open + return false, errno +} + +// GetDeviceNameFromMount when given a mount point, finds the device name from its global +// mount point +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device name from /proc/mounts in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + klog.V(4).Infof("Directory %s is not mounted", mountPath) + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := path.Join(pluginDir, MountsInGlobalPDPath) + for _, ref := range refs { + if strings.HasPrefix(ref, basemountPath) { + volumeID, err := filepath.Rel(basemountPath, ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +func listProcMounts(mountFilePath string) ([]MntPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MntPoint, error) { + out := []MntPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) + } + + mp := MntPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +// MakeRShared makes the bind mount rshared if needed. +func (mounter *Mounter) MakeRShared(path string) error { + return doMakeRShared(path, procMountInfoPath) +} + +// GetFileType returns the string FileType of the path. +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// MakeDir creates a directory at the specified path. +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// MakeFile creates a file with permissions 0644 at the specified path. +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + if err != nil { + if !os.IsExist(err) { + return err + } + defer f.Close() + } + return nil +} + +// ExistsPath returns true if the specified path exists. +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +// EvalHostSymlinks returns the actual path a symlink points to. +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + + options = append(options, "defaults") + + if !readOnly { + // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Run("fsck", args...) + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warning("'fsck' not found on system; continuing mount without running 'fsck'") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + } + + // Try to mount the disk + klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) + mountErr := mounter.Interface.Mount(source, target, fstype, options) + if mountErr != nil { + // Mount failed. This indicates either that the disk is unformatted or + // it contains an unexpected filesystem. + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return err + } + if existingFormat == "" { + if readOnly { + // Don't attempt to format if mounting as readonly, return an error to reflect this. + return errors.New("failed to mount unformatted volume as read only") + } + + // Disk is unformatted so format it. + args := []string{source} + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + _, err := mounter.Exec.Run("mkfs."+fstype, args...) + if err == nil { + // the disk has been formatted successfully try to mount it again. + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + return mounter.Interface.Mount(source, target, fstype, options) + } + klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) + return err + } + // Disk is already formatted and failed to mount + if len(fstype) == 0 || fstype == existingFormat { + // This is mount error + return mountErr + } + // Block device is formatted with unexpected filesystem, let the user know + return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) + } + return mountErr +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Run("blkid", args...) + output := string(dataOut) + klog.V(4).Infof("Output: %q, err: %v", output, err) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// isShared returns true, if given path is on a mount point that has shared +// mount propagation. +func isShared(mount string, mountInfoPath string) (bool, error) { + info, err := findMountInfo(mount, mountInfoPath) + if err != nil { + return false, err + } + + // parse optional parameters + for _, opt := range info.optionalFields { + if strings.HasPrefix(opt, "shared:") { + return true, nil + } + } + return false, nil +} + +// This represents a single line in /proc//mountinfo. +type mountInfo struct { + // Unique ID for the mount (maybe reused after umount). + id int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + parentID int + // The value of `st_dev` for files on this filesystem. + majorMinor string + // The pathname of the directory in the filesystem which forms the root of this mount. + root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + source string + // Mount point, the pathname of the mount point. + mountPoint string + // Optional fields, zero or more fields of the form "tag[:value]". + optionalFields []string + // The filesystem type in the form "type[.subtype]". + fsType string + // Per-mount options. + mountOptions []string + // Per-superblock options. + superOptions []string +} + +// parseMountInfo parses /proc/xxx/mountinfo. +func parseMountInfo(filename string) ([]mountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []mountInfo{}, err + } + contentStr := string(content) + infos := []mountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + info := mountInfo{ + id: id, + parentID: parentID, + majorMinor: fields[2], + root: fields[3], + mountPoint: fields[4], + mountOptions: strings.Split(fields[5], ","), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.optionalFields = append(info.optionalFields, fields[i]) + } + // Parse the rest 3 fields. + i++ + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.fsType = fields[i] + info.source = fields[i+1] + info.superOptions = strings.Split(fields[i+2], ",") + infos = append(infos, info) + } + return infos, nil +} + +func findMountInfo(path, mountInfoPath string) (mountInfo, error) { + infos, err := parseMountInfo(mountInfoPath) + if err != nil { + return mountInfo{}, err + } + + // process /proc/xxx/mountinfo in backward order and find the first mount + // point that is prefix of 'path' - that's the mount where path resides + var info *mountInfo + for i := len(infos) - 1; i >= 0; i-- { + if PathWithinBase(path, infos[i].mountPoint) { + info = &infos[i] + break + } + } + if info == nil { + return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path) + } + return *info, nil +} + +// doMakeRShared is common implementation of MakeRShared on Linux. It checks if +// path is shared and bind-mounts it as rshared if needed. mountCmd and +// mountArgs are expected to contain mount-like command, doMakeRShared will add +// '--bind ' and '--make-rshared ' to mountArgs. +func doMakeRShared(path string, mountInfoFilename string) error { + shared, err := isShared(path, mountInfoFilename) + if err != nil { + return err + } + if shared { + klog.V(4).Infof("Directory %s is already on a shared mount", path) + return nil + } + + klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) + // mount --bind /var/lib/kubelet /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { + return fmt.Errorf("failed to bind-mount %s: %v", path, err) + } + + // mount --make-rshared /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil { + return fmt.Errorf("failed to make %s rshared: %v", path, err) + } + + return nil +} + +// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux. +func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) { + info, err := findMountInfo(path, mountInfoFilename) + if err != nil { + return false, err + } + + // "seclabel" can be both in mount options and super options. + for _, opt := range info.superOptions { + if opt == "seclabel" { + return true, nil + } + } + for _, opt := range info.mountOptions { + if opt == "seclabel" { + return true, nil + } + } + return false, nil +} + +// PrepareSafeSubpath safely creates a new subpath. +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(mounter, subPath) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +// This implementation is shared between Linux and NsEnterMounter +func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { + if !PathWithinBase(subpath.Path, subpath.VolumePath) { + return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) + } + fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) + if err != nil { + return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err) + } + return fd, nil +} + +// prepareSubpathTarget creates target for bind-mount of subpath. It returns +// "true" when the target already exists and something is mounted there. +// Given Subpath must have all paths with already resolved symlinks and with +// paths relevant to kubelet (when it runs in a container). +// This function is called also by NsEnterMounter. It works because +// /var/lib/kubelet is mounted from the host into the container with Kubelet as +// /var/lib/kubelet too. +func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) { + // Early check for already bind-mounted subpath. + bindPathTarget := getSubpathBindTarget(subpath) + notMount, err := IsNotMountPoint(mounter, bindPathTarget) + if err != nil { + if !os.IsNotExist(err) { + return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) + } + // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. + notMount = true + } + if !notMount { + // It's already mounted + klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + return true, bindPathTarget, nil + } + + // bindPathTarget is in /var/lib/kubelet and thus reachable without any + // translation even to containerized kubelet. + bindParent := filepath.Dir(bindPathTarget) + err = os.MkdirAll(bindParent, 0750) + if err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err) + } + + t, err := os.Lstat(subpath.Path) + if err != nil { + return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) + } + + if t.Mode()&os.ModeDir > 0 { + if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) + } + } else { + // "/bin/touch ". + // A file is enough for all possible targets (symlink, device, pipe, + // socket, ...), bind-mounting them into a file correctly changes type + // of the target file. + if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { + return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) + } + } + return false, bindPathTarget, nil +} + +func getSubpathBindTarget(subpath Subpath) string { + // containerName is DNS label, i.e. safe as a directory name. + return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) +} + +func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) { + // Linux, kubelet runs on the host: + // - safely open the subpath + // - bind-mount /proc//fd/ to subpath target + // User can't change /proc//fd/ to point to a bad place. + + // Evaluate all symlinks here once for all subsequent functions. + newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) + } + newPath, err := filepath.EvalSymlinks(subpath.Path) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) + } + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) + subpath.VolumePath = newVolumePath + subpath.Path = newPath + + fd, err := safeOpenSubPath(mounter, subpath) + if err != nil { + return "", err + } + defer syscall.Close(fd) + + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + if err != nil { + return "", err + } + if alreadyMounted { + return bindPathTarget, nil + } + + success := false + defer func() { + // Cleanup subpath on error + if !success { + klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + kubeletPid := os.Getpid() + mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) + + // Do the bind mount + options := []string{"bind"} + klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { + return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) + } + success = true + + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +// CleanSubPaths cleans up the subpath mounts for the volume in the pod directory. +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +// This implementation is shared between Linux and NsEnterMounter +func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { + // scan /var/lib/kubelet/pods//volume-subpaths//* + subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) + klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) + + containerDirs, err := ioutil.ReadDir(subPathDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("error reading %s: %s", subPathDir, err) + } + + for _, containerDir := range containerDirs { + if !containerDir.IsDir() { + klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + continue + } + klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + + // scan /var/lib/kubelet/pods//volume-subpaths///* + fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) + subPaths, err := ioutil.ReadDir(fullContainerDirPath) + if err != nil { + return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err) + } + for _, subPath := range subPaths { + if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil { + return err + } + } + // Whole container has been processed, remove its directory. + if err := os.Remove(fullContainerDirPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) + } + klog.V(5).Infof("Removed %s", fullContainerDirPath) + } + // Whole pod volume subpaths have been cleaned up, remove its subpath directory. + if err := os.Remove(subPathDir); err != nil { + return fmt.Errorf("error deleting %s: %s", subPathDir, err) + } + klog.V(5).Infof("Removed %s", subPathDir) + + // Remove entire subpath directory if it's the last one + podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) + if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { + return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) + } + klog.V(5).Infof("Removed %s", podSubPathDir) + return nil +} + +// doCleanSubPath tears down the single subpath bind mount +func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { + // process /var/lib/kubelet/pods//volume-subpaths/// + klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) + notMnt, err := IsNotMountPoint(mounter, fullSubPath) + if err != nil { + return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err) + } + // Unmount it + if !notMnt { + if err = mounter.Unmount(fullSubPath); err != nil { + return fmt.Errorf("error unmounting %s: %s", fullSubPath, err) + } + klog.V(5).Infof("Unmounted %s", fullSubPath) + } + // Remove it *non*-recursively, just in case there were some hiccups. + if err = os.Remove(fullSubPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullSubPath, err) + } + klog.V(5).Infof("Removed %s", fullSubPath) + return nil +} + +// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty +func cleanSubPath(mounter Interface, subpath Subpath) error { + containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + + // Clean subdir bindmount + if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { + return err + } + + // Recursively remove directories if empty + if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { + return err + } + + return nil +} + +// removeEmptyDirs works backwards from endDir to baseDir and removes each directory +// if it is empty. It stops once it encounters a directory that has content +func removeEmptyDirs(baseDir, endDir string) error { + if !PathWithinBase(endDir, baseDir) { + return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) + } + + for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { + s, err := os.Stat(curDir) + if err != nil { + if os.IsNotExist(err) { + klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + continue + } + return fmt.Errorf("error stat %q: %v", curDir, err) + } + if !s.IsDir() { + return fmt.Errorf("path %q not a directory", curDir) + } + + err = os.Remove(curDir) + if os.IsExist(err) { + klog.V(5).Infof("Directory %q not empty, not removing", curDir) + break + } else if err != nil { + return fmt.Errorf("error removing directory %q: %v", curDir, err) + } + klog.V(5).Infof("Removed directory %q", curDir) + } + return nil +} + +// SafeMakeDir safely creates a subdirectory at the specified path base +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + + return doSafeMakeDir(realFullPath, realBase, perm) +} + +// GetMountRefs finds all of the mount references to the source. +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(pathname); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return searchMountPoints(realpath, procMountInfoPath) +} + +// GetSELinuxSupport is common implementation of GetSELinuxSupport on Linux. +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return getSELinuxSupport(pathname, procMountInfoPath) +} + +// GetFSGroup returns the filesystem group for the path name. +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return 0, err + } + return getFSGroup(realpath) +} + +// GetMode returns the unix filesystem mode for the path. +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return getMode(pathname) +} + +// This implementation is shared between Linux and NsEnterMounter +func getFSGroup(pathname string) (int64, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return int64(info.Sys().(*syscall.Stat_t).Gid), nil +} + +// This implementation is shared between Linux and NsEnterMounter +func getMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// This implementation is shared between Linux and NsEnterMounter. Both pathname +// and base must be either already resolved symlinks or that will be resolved in +// kubelet's mount namespace (in case it runs containerized). +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", existingPath, err) + } + if !PathWithinBase(fullExistingPath, base) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + parentFD, err := doSafeOpen(fullExistingPath, base) + if err != nil { + return fmt.Errorf("cannot open directory %s: %s", existingPath, err) + } + childFD := -1 + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + } + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink. + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) + if err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + // Dive into the created directory + childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) + if err != nil { + return fmt.Errorf("cannot open %s: %s", currentPath, err) + } + // We can be sure that childFD is safe to use. It could be changed + // by user after Mkdirat() and before Openat(), however: + // - it could not be changed to symlink - we use nofollowFlags + // - it could be changed to a file (or device, pipe, socket, ...) + // but either subsequent Mkdirat() fails or we mount this file + // to user's container. Security is no violated in both cases + // and user either gets error or the file that it can already access. + + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + parentFD = childFD + childFD = -1 + } + + // Everything was created. mkdirat(..., perm) above was affected by current + // umask and we must apply the right permissions to the last directory + // (that's the one that will be available to the container as subpath) + // so user can read/write it. This is the behavior of previous code. + // TODO: chmod all created directories, not just the last one. + // parentFD is the last created directory. + + // Translate perm (os.FileMode) to uint32 that fchmod() expects + kernelPerm := uint32(perm & os.ModePerm) + if perm&os.ModeSetgid > 0 { + kernelPerm |= syscall.S_ISGID + } + if perm&os.ModeSetuid > 0 { + kernelPerm |= syscall.S_ISUID + } + if perm&os.ModeSticky > 0 { + kernelPerm |= syscall.S_ISVTX + } + if err = syscall.Fchmod(parentFD, kernelPerm); err != nil { + return fmt.Errorf("chmod %q failed: %s", currentPath, err) + } + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + dirs := strings.Split(rel, string(filepath.Separator)) + + // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. + // This should be faster than looping through all dirs and calling os.Stat() + // on each of them, as the symlinks are resolved only once with OpenAt(). + currentPath := base + fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0) + if err != nil { + return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) + } + defer func() { + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + }() + for i, dir := range dirs { + // Using O_PATH here will prevent hangs in case user replaces directory with + // fifo + childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0) + if err != nil { + if os.IsNotExist(err) { + return currentPath, dirs[i:], nil + } + return base, nil, err + } + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + fd = childFD + currentPath = filepath.Join(currentPath, dir) + } + return pathname, []string{}, nil +} + +// This implementation is shared between Linux and NsEnterMounter +// Open path and return its fd. +// Symlinks are disallowed (pathname must already resolve symlinks), +// and the path must be within the base directory. +func doSafeOpen(pathname string, base string) (int, error) { + pathname = filepath.Clean(pathname) + base = filepath.Clean(base) + + // Calculate segments to follow + subpath, err := filepath.Rel(base, pathname) + if err != nil { + return -1, err + } + segments := strings.Split(subpath, string(filepath.Separator)) + + // Assumption: base is the only directory that we have under control. + // Base dir is not allowed to be a symlink. + parentFD, err := syscall.Open(base, nofollowFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open directory %s: %s", base, err) + } + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + } + } + }() + + childFD := -1 + defer func() { + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := base + + // Follow the segments one by one using openat() to make + // sure the user cannot change already existing directories into symlinks. + for _, seg := range segments { + currentPath = filepath.Join(currentPath, seg) + if !PathWithinBase(currentPath, base) { + return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) + } + + klog.V(5).Infof("Opening path %s", currentPath) + childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) + } + + var deviceStat unix.Stat_t + err := unix.Fstat(childFD, &deviceStat) + if err != nil { + return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err) + } + fileFmt := deviceStat.Mode & syscall.S_IFMT + if fileFmt == syscall.S_IFLNK { + return -1, fmt.Errorf("Unexpected symlink found %s", currentPath) + } + + // Close parentFD + if err = syscall.Close(parentFD); err != nil { + return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) + } + // Set child to new parent + parentFD = childFD + childFD = -1 + } + + // We made it to the end, return this fd, don't close it + finalFD := parentFD + parentFD = -1 + + return finalFD, nil +} + +// searchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// This function assumes source cannot be device. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := parseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + majorMinor := "" + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].id + rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint)) + majorMinor = mis[i].majorMinor + break + } + } + + if rootPath == "" || majorMinor == "" { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].id == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].root == rootPath && mis[i].majorMinor == majorMinor { + refs = append(refs, mis[i].mountPoint) + } + } + + return refs, nil +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go new file mode 100644 index 00000000000..cab7aecc3ba --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go @@ -0,0 +1,162 @@ +// +build !linux,!windows,!providerless + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "os" +) + +// Mounter provides the default unsupported implementation of mount.Interface +type Mounter struct { + mounterPath string +} + +var errUnsupported = errors.New(" util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount returns an error +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return errUnsupported +} + +// Unmount returns an error +func (mounter *Mounter) Unmount(target string) error { + return errUnsupported +} + +// List returns an error +func (mounter *Mounter) List() ([]MntPoint, error) { + return []MntPoint{}, errUnsupported +} + +// IsMountPointMatch returns an error +func (mounter *Mounter) IsMountPointMatch(mp MntPoint, dir string) bool { + return (mp.Path == dir) +} + +// IsNotMountPoint returns an error +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint returns an error +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, errUnsupported +} + +// GetDeviceNameFromMount returns an error +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return "", errUnsupported +} + +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + return "", errUnsupported +} + +// DeviceOpened returns an error +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, errUnsupported +} + +// PathIsDevice returns an error +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return true, errUnsupported +} + +// MakeRShared returns an error +func (mounter *Mounter) MakeRShared(path string) error { + return errUnsupported +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, errUnsupported +} + +// GetFileType returns an error +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return FileType("fake"), errUnsupported +} + +// MakeDir returns an error +func (mounter *Mounter) MakeDir(pathname string) error { + return errUnsupported +} + +// MakeFile returns an error +func (mounter *Mounter) MakeFile(pathname string) error { + return errUnsupported +} + +// ExistsPath returns an error +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return true, errors.New("not implemented") +} + +// EvalHostSymlinks returns an error +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return "", errUnsupported +} + +// PrepareSafeSubpath returns an error +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, errUnsupported +} + +// CleanSubPaths returns an error +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return errUnsupported +} + +// SafeMakeDir returns an error +func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return errUnsupported +} + +// GetMountRefs returns an error +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +// GetFSGroup returns an error +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + +// GetSELinuxSupport returns an error +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return false, errors.New("not implemented") +} + +// GetMode returns an error +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return 0, errors.New("not implemented") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go new file mode 100644 index 00000000000..18fb28417c7 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go @@ -0,0 +1,613 @@ +// +build windows,!providerless + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog" + + utilpath "k8s.io/utils/path" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount : mounts source to target as NTFS with given options. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + target = normalizeWindowsPath(target) + + if source == "tmpfs" { + klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + options, source, target, fstype) + bindSource := "" + + // tell it's going to mount azure disk or azure file according to options + if bind, _, _ := isBind(options); bind { + // mount azure disk + bindSource = normalizeWindowsPath(source) + } else { + if len(options) < 2 { + klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + options, len(options), source, target) + return nil + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) + } + + bindSource = source + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`) + + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", options[0]), + fmt.Sprintf("smbpassword=%s", options[1]), + fmt.Sprintf("smbremotepath=%s", source)) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output)) + } + } + + if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + + return nil +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = normalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MntPoint, error) { + return []MntPoint{}, nil +} + +// IsMountPointMatch determines if the mountpoint matches the dir +func (mounter *Mounter) IsMountPointMatch(mp MntPoint, dir string) bool { + return mp.Path == dir +} + +// IsNotMountPoint determines if a directory is a mountpoint. +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + // If current file is a symlink, then it is a mountpoint. + if stat.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(file) + if err != nil { + return true, fmt.Errorf("readlink error: %v", err) + } + exists, err := mounter.ExistsPath(target) + if err != nil { + return true, err + } + return !exists, nil + } + + return true, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device(drive) name in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath)) + for _, ref := range refs { + if strings.Contains(ref, basemountPath) { + volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +// DeviceOpened determines if the device is in use elsewhere +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, nil +} + +// PathIsDevice determines if a path is a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return false, nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. Empty implementation here. +func (mounter *Mounter) MakeRShared(path string) error { + return nil +} + +// GetFileType checks for sockets/block/character devices +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// MakeDir creates a new directory +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// MakeFile creates an empty file +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// ExistsPath checks whether the path exists +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +// EvalHostSymlinks returns the path name after evaluating symlinks +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// check whether hostPath is within volume path +// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started +func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(hostPath) == 0 { + return []uintptr{}, nil + } + + finalSubPath, err := filepath.EvalSymlinks(hostPath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err) + } + finalVolumePath, err := filepath.EvalSymlinks(volumePath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err) + } + + return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath) +} + +// lock all intermediate subPath directories and check they are all within volumePath +// volumePath & subPath should not contain any symlink, otherwise it will return error +func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(subPath) == 0 { + return []uintptr{}, nil + } + + // get relative path to volumePath + relSubPath, err := filepath.Rel(volumePath, subPath) + if err != nil { + return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) + } + if startsWithBackstep(relSubPath) { + return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) + } + + if relSubPath == "." { + // volumePath and subPath are equal + return []uintptr{}, nil + } + + fileHandles := []uintptr{} + var errorResult error + + currentFullPath := volumePath + dirs := strings.Split(relSubPath, string(os.PathSeparator)) + for _, dir := range dirs { + // lock intermediate subPath directory first + currentFullPath = filepath.Join(currentFullPath, dir) + handle, err := lockPath(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err) + break + } + fileHandles = append(fileHandles, handle) + + // make sure intermediate subPath directory does not contain symlink any more + stat, err := os.Lstat(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err) + break + } + if stat.Mode()&os.ModeSymlink != 0 { + errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath) + break + } + + if !PathWithinBase(currentFullPath, volumePath) { + errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) + break + } + } + + return fileHandles, errorResult +} + +// unlockPath unlock directories +func unlockPath(fileHandles []uintptr) { + if fileHandles != nil { + for _, handle := range fileHandles { + syscall.CloseHandle(syscall.Handle(handle)) + } + } +} + +// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path +func lockPath(path string) (uintptr, error) { + if len(path) == 0 { + return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return uintptr(syscall.InvalidHandle), err + } + access := uint32(syscall.GENERIC_READ) + sharemode := uint32(syscall.FILE_SHARE_READ) + createmode := uint32(syscall.OPEN_EXISTING) + flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT) + fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0) + return uintptr(fd), err +} + +// PrepareSafeSubpath locks all directories in subPath and check they're not symlinks. +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) + + // Unlock the directories when the container starts + cleanupAction = func() { + unlockPath(handles) + } + return subPath.Path, cleanupAction, err +} + +// CleanSubPaths no bind-mounts for subpaths are necessary on Windows +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) + if err != nil { + return err + } + driverPath := driveLetter + ":" + target = normalizeWindowsPath(target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { + klog.Errorf("mklink failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +func normalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// Get drive letter according to windows disk number +func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) { + cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) + } + if len(string(output)) < 1 { + return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") + } + return string(output)[:1], nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + return []string{pathname}, nil +} + +// GetFSGroup always returns 0. We actually don't set FSGroup on +// windows platform, see SetVolumeOwnership implementation. +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return 0, nil +} + +// GetSELinuxSupport returns false +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + // Windows does not support SELinux. + return false, nil +} + +// GetMode returns the filesystem mode. +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + if len(toCreate) == 0 { + return nil + } + + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening existing directory %s: %s", existingPath, err) + } + fullBasePath, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("cannot read link %s: %s", base, err) + } + if !PathWithinBase(fullExistingPath, fullBasePath) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + // lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom) + fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath) + defer unlockPath(fileHandles) + if err != nil { + return err + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink by lock that directory immediately + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + if err := os.Mkdir(currentPath, perm); err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + handle, err := lockPath(currentPath) + if err != nil { + return fmt.Errorf("cannot lock path %s: %s", currentPath, err) + } + defer syscall.CloseHandle(syscall.Handle(handle)) + // make sure newly created directory does not contain symlink after lock + stat, err := os.Lstat(currentPath) + if err != nil { + return fmt.Errorf("Lstat(%q) error: %v", currentPath, err) + } + if stat.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath) + } + } + + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + + if startsWithBackstep(rel) { + return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) + } + + if rel == "." { + // base and pathname are equal + return pathname, []string{}, nil + } + + dirs := strings.Split(rel, string(filepath.Separator)) + + parent := base + currentPath := base + for i, dir := range dirs { + parent = currentPath + currentPath = filepath.Join(parent, dir) + if _, err := os.Lstat(currentPath); err != nil { + if os.IsNotExist(err) { + return parent, dirs[i:], nil + } + return base, nil, err + } + } + + return pathname, []string{}, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8363ac01ecc..11c569833be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1831,6 +1831,8 @@ k8s.io/legacy-cloud-providers/aws k8s.io/legacy-cloud-providers/azure k8s.io/legacy-cloud-providers/azure/auth k8s.io/legacy-cloud-providers/gce +k8s.io/legacy-cloud-providers/openstack +k8s.io/legacy-cloud-providers/openstack/util/mount k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere/testing k8s.io/legacy-cloud-providers/vsphere/vclib