mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #80027 from ramineni/staging-openstack-provider
Move Openstack provider to staging
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
e0e7fc2201
2
go.mod
2
go.mod
@ -71,7 +71,6 @@ require (
|
||||
github.com/google/go-cmp v0.3.0
|
||||
github.com/google/gofuzz v1.0.0
|
||||
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d
|
||||
github.com/gophercloud/gophercloud v0.1.0
|
||||
github.com/gorilla/context v1.1.1 // indirect
|
||||
github.com/gorilla/mux v1.7.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
@ -91,7 +90,6 @@ require (
|
||||
github.com/miekg/dns v1.1.4
|
||||
github.com/mindprince/gonvml v0.0.0-20171110221305-fee913ce8fb2 // indirect
|
||||
github.com/mistifyio/go-zfs v2.1.1+incompatible // indirect
|
||||
github.com/mitchellh/mapstructure v1.1.2
|
||||
github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb // indirect
|
||||
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
|
||||
github.com/mrunalp/fileutils v0.0.0-20160930181131-4ee1cc9a8058 // indirect
|
||||
|
||||
@ -16,10 +16,10 @@ go_library(
|
||||
"//cmd/kubelet/app:__pkg__",
|
||||
],
|
||||
deps = [
|
||||
"//pkg/cloudprovider/providers/openstack:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/vsphere:go_default_library",
|
||||
],
|
||||
)
|
||||
@ -33,10 +33,7 @@ filegroup(
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//pkg/cloudprovider/providers/openstack:all-srcs",
|
||||
],
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
)
|
||||
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package openstack
|
||||
@ -20,9 +20,9 @@ package cloudprovider
|
||||
|
||||
import (
|
||||
// Cloud providers
|
||||
_ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
|
||||
_ "k8s.io/legacy-cloud-providers/aws"
|
||||
_ "k8s.io/legacy-cloud-providers/azure"
|
||||
_ "k8s.io/legacy-cloud-providers/gce"
|
||||
_ "k8s.io/legacy-cloud-providers/openstack"
|
||||
_ "k8s.io/legacy-cloud-providers/vsphere"
|
||||
)
|
||||
|
||||
@ -17,7 +17,6 @@ go_library(
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/volume/cinder",
|
||||
deps = [
|
||||
"//pkg/cloudprovider/providers/openstack:go_default_library",
|
||||
"//pkg/features:go_default_library",
|
||||
"//pkg/util/mount:go_default_library",
|
||||
"//pkg/volume:go_default_library",
|
||||
@ -33,6 +32,7 @@ go_library(
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/cloud-provider:go_default_library",
|
||||
"//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
"//vendor/k8s.io/utils/keymutex:go_default_library",
|
||||
@ -49,7 +49,6 @@ go_test(
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//pkg/cloudprovider/providers/openstack:go_default_library",
|
||||
"//pkg/util/mount:go_default_library",
|
||||
"//pkg/volume:go_default_library",
|
||||
"//pkg/volume/testing:go_default_library",
|
||||
@ -60,6 +59,7 @@ go_test(
|
||||
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
|
||||
"//staging/src/k8s.io/cloud-provider:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@ -25,18 +25,18 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
cloudprovider "k8s.io/cloud-provider"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
"k8s.io/kubernetes/pkg/volume/util"
|
||||
"k8s.io/legacy-cloud-providers/openstack"
|
||||
"k8s.io/utils/keymutex"
|
||||
utilstrings "k8s.io/utils/strings"
|
||||
)
|
||||
|
||||
@ -28,11 +28,11 @@ import (
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utiltesting "k8s.io/client-go/util/testing"
|
||||
"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
volumetest "k8s.io/kubernetes/pkg/volume/testing"
|
||||
"k8s.io/kubernetes/pkg/volume/util"
|
||||
"k8s.io/legacy-cloud-providers/openstack"
|
||||
)
|
||||
|
||||
func TestCanSupport(t *testing.T) {
|
||||
|
||||
@ -12,6 +12,7 @@ filegroup(
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/aws:all-srcs",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/azure:all-srcs",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/gce:all-srcs",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack:all-srcs",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/vsphere:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
|
||||
@ -14,6 +14,8 @@ require (
|
||||
github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534
|
||||
github.com/aws/aws-sdk-go v1.16.26
|
||||
github.com/dnaeon/go-vcr v1.0.1 // indirect
|
||||
github.com/gophercloud/gophercloud v0.1.0
|
||||
github.com/mitchellh/mapstructure v1.1.2
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c
|
||||
github.com/satori/go.uuid v1.2.0 // indirect
|
||||
@ -21,6 +23,7 @@ require (
|
||||
github.com/vmware/govmomi v0.20.1
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
|
||||
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f
|
||||
google.golang.org/api v0.6.1-0.20190607001116-5213b8090861
|
||||
gopkg.in/gcfg.v1 v1.2.0
|
||||
gopkg.in/warnings.v0 v0.1.1 // indirect
|
||||
|
||||
5
staging/src/k8s.io/legacy-cloud-providers/go.sum
generated
5
staging/src/k8s.io/legacy-cloud-providers/go.sum
generated
@ -100,6 +100,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
|
||||
github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
|
||||
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o=
|
||||
github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
@ -112,6 +113,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
|
||||
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
@ -135,6 +137,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@ -173,6 +177,7 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
|
||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
|
||||
@ -9,7 +9,6 @@ load(
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"doc.go",
|
||||
"metadata.go",
|
||||
"openstack.go",
|
||||
"openstack_client.go",
|
||||
@ -19,9 +18,9 @@ go_library(
|
||||
"openstack_routes.go",
|
||||
"openstack_volumes.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack",
|
||||
importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack",
|
||||
importpath = "k8s.io/legacy-cloud-providers/openstack",
|
||||
deps = [
|
||||
"//pkg/util/mount:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
@ -40,6 +39,7 @@ go_library(
|
||||
"//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:go_default_library",
|
||||
"//vendor/github.com/gophercloud/gophercloud:go_default_library",
|
||||
"//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
|
||||
"//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library",
|
||||
@ -103,6 +103,9 @@ filegroup(
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
)
|
||||
@ -3,4 +3,4 @@
|
||||
* [Angus Lees](https://github.com/anguslees)
|
||||
|
||||
|
||||
[]()
|
||||
[]()
|
||||
@ -30,7 +30,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
osmount "k8s.io/legacy-cloud-providers/openstack/util/mount"
|
||||
"k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
@ -125,7 +125,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) {
|
||||
|
||||
klog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)
|
||||
|
||||
mounter := mount.New("" /* default mount path */)
|
||||
mounter := osmount.New("" /* default mount path */)
|
||||
err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
|
||||
if err != nil {
|
||||
err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})
|
||||
@ -0,0 +1,43 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"mount.go",
|
||||
"mount_linux.go",
|
||||
"mount_unsupported.go",
|
||||
"mount_windows.go",
|
||||
],
|
||||
importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack/util/mount",
|
||||
importpath = "k8s.io/legacy-cloud-providers/openstack/util/mount",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = select({
|
||||
"@io_bazel_rules_go//go/platform:linux": [
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/golang.org/x/sys/unix:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
"//vendor/k8s.io/utils/io:go_default_library",
|
||||
"//vendor/k8s.io/utils/path:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows": [
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
"//vendor/k8s.io/utils/path:go_default_library",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
@ -0,0 +1,380 @@
|
||||
// +build !providerless
|
||||
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mount provides a linux-centric library for mounting filesystems/
|
||||
// Vendored for the OpenStack provider and will be removed in 1.17
|
||||
package mount
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FileType is one of the types of supported Linux file types.
|
||||
type FileType string
|
||||
|
||||
// A complete list of the support file types in the mount package.
|
||||
const (
|
||||
// Default mount command if mounter path is not specified
|
||||
defaultMountCommand = "mount"
|
||||
MountsInGlobalPDPath = "mounts"
|
||||
FileTypeDirectory FileType = "Directory"
|
||||
FileTypeFile FileType = "File"
|
||||
FileTypeSocket FileType = "Socket"
|
||||
FileTypeCharDev FileType = "CharDevice"
|
||||
FileTypeBlockDev FileType = "BlockDevice"
|
||||
)
|
||||
|
||||
// Interface is the general interface to filesystem mounts.
|
||||
type Interface interface {
|
||||
// Mount mounts source to target as fstype with given options.
|
||||
Mount(source string, target string, fstype string, options []string) error
|
||||
// Unmount unmounts given target.
|
||||
Unmount(target string) error
|
||||
// List returns a list of all mounted filesystems. This can be large.
|
||||
// On some platforms, reading mounts is not guaranteed consistent (i.e.
|
||||
// it could change between chunked reads). This is guaranteed to be
|
||||
// consistent.
|
||||
List() ([]MntPoint, error)
|
||||
// IsMountPointMatch determines if the mountpoint matches the dir
|
||||
IsMountPointMatch(mp MntPoint, dir string) bool
|
||||
// IsNotMountPoint determines if a directory is a mountpoint.
|
||||
// It should return ErrNotExist when the directory does not exist.
|
||||
// IsNotMountPoint is more expensive than IsLikelyNotMountPoint.
|
||||
// IsNotMountPoint detects bind mounts in linux.
|
||||
// IsNotMountPoint enumerates all the mountpoints using List() and
|
||||
// the list of mountpoints may be large, then it uses
|
||||
// IsMountPointMatch to evaluate whether the directory is a mountpoint
|
||||
IsNotMountPoint(file string) (bool, error)
|
||||
// IsLikelyNotMountPoint uses heuristics to determine if a directory
|
||||
// is a mountpoint.
|
||||
// It should return ErrNotExist when the directory does not exist.
|
||||
// IsLikelyNotMountPoint does NOT properly detect all mountpoint types
|
||||
// most notably linux bind mounts.
|
||||
IsLikelyNotMountPoint(file string) (bool, error)
|
||||
// DeviceOpened determines if the device is in use elsewhere
|
||||
// on the system, i.e. still mounted.
|
||||
DeviceOpened(pathname string) (bool, error)
|
||||
// PathIsDevice determines if a path is a device.
|
||||
PathIsDevice(pathname string) (bool, error)
|
||||
// GetDeviceNameFromMount finds the device name by checking the mount path
|
||||
// to get the global mount path which matches its plugin directory
|
||||
GetDeviceNameFromMount(mountPath, pluginDir string) (string, error)
|
||||
// MakeRShared checks that given path is on a mount with 'rshared' mount
|
||||
// propagation. If not, it bind-mounts the path as rshared.
|
||||
MakeRShared(path string) error
|
||||
// GetFileType checks for file/directory/socket/block/character devices.
|
||||
// Will operate in the host mount namespace if kubelet is running in a container
|
||||
GetFileType(pathname string) (FileType, error)
|
||||
// MakeFile creates an empty file.
|
||||
// Will operate in the host mount namespace if kubelet is running in a container
|
||||
MakeFile(pathname string) error
|
||||
// MakeDir creates a new directory.
|
||||
// Will operate in the host mount namespace if kubelet is running in a container
|
||||
MakeDir(pathname string) error
|
||||
// SafeMakeDir creates subdir within given base. It makes sure that the
|
||||
// created directory does not escape given base directory mis-using
|
||||
// symlinks. Note that the function makes sure that it creates the directory
|
||||
// somewhere under the base, nothing else. E.g. if the directory already
|
||||
// exists, it may exist outside of the base due to symlinks.
|
||||
// This method should be used if the directory to create is inside volume
|
||||
// that's under user control. User must not be able to use symlinks to
|
||||
// escape the volume to create directories somewhere else.
|
||||
SafeMakeDir(subdir string, base string, perm os.FileMode) error
|
||||
// Will operate in the host mount namespace if kubelet is running in a container.
|
||||
// Error is returned on any other error than "file not found".
|
||||
ExistsPath(pathname string) (bool, error)
|
||||
// EvalHostSymlinks returns the path name after evaluating symlinks.
|
||||
// Will operate in the host mount namespace if kubelet is running in a container.
|
||||
EvalHostSymlinks(pathname string) (string, error)
|
||||
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
|
||||
// pod volume directory.
|
||||
CleanSubPaths(podDir string, volumeName string) error
|
||||
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
|
||||
// that's 1) inside given volumePath and 2) immutable after this call.
|
||||
//
|
||||
// newHostPath - location of prepared subPath. It should be used instead of
|
||||
// hostName when running the container.
|
||||
// cleanupAction - action to run when the container is running or it failed to start.
|
||||
//
|
||||
// CleanupAction must be called immediately after the container with given
|
||||
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
|
||||
// when the pod finishes.
|
||||
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
|
||||
// GetMountRefs finds all mount references to the path, returns a
|
||||
// list of paths. Path could be a mountpoint path, device or a normal
|
||||
// directory (for bind mount).
|
||||
GetMountRefs(pathname string) ([]string, error)
|
||||
// GetFSGroup returns FSGroup of the path.
|
||||
GetFSGroup(pathname string) (int64, error)
|
||||
// GetSELinuxSupport returns true if given path is on a mount that supports
|
||||
// SELinux.
|
||||
GetSELinuxSupport(pathname string) (bool, error)
|
||||
// GetMode returns permissions of the path.
|
||||
GetMode(pathname string) (os.FileMode, error)
|
||||
}
|
||||
|
||||
// Subpath is the string path of the volume mount.
|
||||
type Subpath struct {
|
||||
// index of the VolumeMount for this container
|
||||
VolumeMountIndex int
|
||||
// Full path to the subpath directory on the host
|
||||
Path string
|
||||
// name of the volume that is a valid directory name.
|
||||
VolumeName string
|
||||
// Full path to the volume path
|
||||
VolumePath string
|
||||
// Path to the pod's directory, including pod UID
|
||||
PodDir string
|
||||
// Name of the container
|
||||
ContainerName string
|
||||
}
|
||||
|
||||
// Exec executes command where mount utilities are. This can be either the host,
|
||||
// container where kubelet runs or even a remote pod with mount utilities.
|
||||
// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does
|
||||
// not provide stdin/stdout/stderr streams.
|
||||
type Exec interface {
|
||||
// Run executes a command and returns its stdout + stderr combined in one
|
||||
// stream.
|
||||
Run(cmd string, args ...string) ([]byte, error)
|
||||
}
|
||||
|
||||
// Compile-time check to ensure all Mounter implementations satisfy
|
||||
// the mount interface
|
||||
var _ Interface = &Mounter{}
|
||||
|
||||
// MntPoint represents a single line in /proc/mounts or /etc/fstab.
|
||||
type MntPoint struct {
|
||||
Device string
|
||||
Path string
|
||||
Type string
|
||||
Opts []string
|
||||
Freq int
|
||||
Pass int
|
||||
}
|
||||
|
||||
// SafeFormatAndMount probes a device to see if it is formatted.
|
||||
// Namely it checks to see if a file system is present. If so it
|
||||
// mounts it otherwise the device is formatted first then mounted.
|
||||
type SafeFormatAndMount struct {
|
||||
Interface
|
||||
Exec
|
||||
}
|
||||
|
||||
// FormatAndMount formats the given disk, if needed, and mounts it.
|
||||
// That is if the disk is not formatted and it is not being mounted as
|
||||
// read-only it will format it first then mount it. Otherwise, if the
|
||||
// disk is already formatted or it is being mounted as read-only, it
|
||||
// will be mounted without formatting.
|
||||
func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error {
|
||||
return mounter.formatAndMount(source, target, fstype, options)
|
||||
}
|
||||
|
||||
// GetDeviceNameFromMount when given a mnt point, find the device from /proc/mounts
|
||||
// returns the device name, reference count, and error code.
|
||||
func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) {
|
||||
mps, err := mounter.List()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
// Find the device name.
|
||||
// FIXME if multiple devices mounted on the same mount path, only the first one is returned
|
||||
device := ""
|
||||
// If mountPath is symlink, need get its target path.
|
||||
slTarget, err := filepath.EvalSymlinks(mountPath)
|
||||
if err != nil {
|
||||
slTarget = mountPath
|
||||
}
|
||||
for i := range mps {
|
||||
if mps[i].Path == slTarget {
|
||||
device = mps[i].Device
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Find all references to the device.
|
||||
refCount := 0
|
||||
for i := range mps {
|
||||
if mps[i].Device == device {
|
||||
refCount++
|
||||
}
|
||||
}
|
||||
return device, refCount, nil
|
||||
}
|
||||
|
||||
// IsNotMountPoint determines if a directory is a mountpoint.
|
||||
// It should return ErrNotExist when the directory does not exist.
|
||||
// This method uses the List() of all mountpoints
|
||||
// It is more extensive than IsLikelyNotMountPoint
|
||||
// and it detects bind mounts in linux
|
||||
func IsNotMountPoint(mounter Interface, file string) (bool, error) {
|
||||
// IsLikelyNotMountPoint provides a quick check
|
||||
// to determine whether file IS A mountpoint
|
||||
notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file)
|
||||
if notMntErr != nil && os.IsPermission(notMntErr) {
|
||||
// We were not allowed to do the simple stat() check, e.g. on NFS with
|
||||
// root_squash. Fall back to /proc/mounts check below.
|
||||
notMnt = true
|
||||
notMntErr = nil
|
||||
}
|
||||
if notMntErr != nil {
|
||||
return notMnt, notMntErr
|
||||
}
|
||||
// identified as mountpoint, so return this fact
|
||||
if notMnt == false {
|
||||
return notMnt, nil
|
||||
}
|
||||
|
||||
// Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts
|
||||
resolvedFile, err := mounter.EvalHostSymlinks(file)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
|
||||
// check all mountpoints since IsLikelyNotMountPoint
|
||||
// is not reliable for some mountpoint types
|
||||
mountPoints, mountPointsErr := mounter.List()
|
||||
if mountPointsErr != nil {
|
||||
return notMnt, mountPointsErr
|
||||
}
|
||||
for _, mp := range mountPoints {
|
||||
if mounter.IsMountPointMatch(mp, resolvedFile) {
|
||||
notMnt = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return notMnt, nil
|
||||
}
|
||||
|
||||
// isBind detects whether a bind mount is being requested and makes the remount options to
|
||||
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
|
||||
// The list equals:
|
||||
// options - 'bind' + 'remount' (no duplicate)
|
||||
func isBind(options []string) (bool, []string, []string) {
|
||||
// Because we have an FD opened on the subpath bind mount, the "bind" option
|
||||
// needs to be included, otherwise the mount target will error as busy if you
|
||||
// remount as readonly.
|
||||
//
|
||||
// As a consequence, all read only bind mounts will no longer change the underlying
|
||||
// volume mount to be read only.
|
||||
bindRemountOpts := []string{"bind", "remount"}
|
||||
bind := false
|
||||
bindOpts := []string{"bind"}
|
||||
|
||||
// _netdev is a userspace mount option and does not automatically get added when
|
||||
// bind mount is created and hence we must carry it over.
|
||||
if checkForNetDev(options) {
|
||||
bindOpts = append(bindOpts, "_netdev")
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
switch option {
|
||||
case "bind":
|
||||
bind = true
|
||||
case "remount":
|
||||
default:
|
||||
bindRemountOpts = append(bindRemountOpts, option)
|
||||
}
|
||||
}
|
||||
|
||||
return bind, bindOpts, bindRemountOpts
|
||||
}
|
||||
|
||||
func checkForNetDev(options []string) bool {
|
||||
for _, option := range options {
|
||||
if option == "_netdev" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HasMountRefs checks the mountPath contains any of the mountRefs.
|
||||
// This is a workaround for the unmount device issue caused by gci mounter.
|
||||
// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
|
||||
// script will cause additional mounts created in the container. Since these mounts are
|
||||
// irrelevant to the original mounts, they should be not considered when checking the
|
||||
// mount references. Current solution is to filter out those mount paths that contain
|
||||
// the string of original mount path.
|
||||
// Plan to work on better approach to solve this issue.
|
||||
func HasMountRefs(mountPath string, mountRefs []string) bool {
|
||||
count := 0
|
||||
for _, ref := range mountRefs {
|
||||
if !strings.Contains(ref, mountPath) {
|
||||
count = count + 1
|
||||
}
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// PathWithinBase checks if give path is within given base directory.
|
||||
func PathWithinBase(fullPath, basePath string) bool {
|
||||
rel, err := filepath.Rel(basePath, fullPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if startsWithBackstep(rel) {
|
||||
// Needed to escape the base path
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// startsWithBackstep checks if the given path starts with a backstep segment
|
||||
func startsWithBackstep(rel string) bool {
|
||||
// normalize to / and check for ../
|
||||
return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
|
||||
}
|
||||
|
||||
// getFileType checks for file/directory/socket and block/character devices
|
||||
func getFileType(pathname string) (FileType, error) {
|
||||
var pathType FileType
|
||||
info, err := os.Stat(pathname)
|
||||
if os.IsNotExist(err) {
|
||||
return pathType, fmt.Errorf("path %q does not exist", pathname)
|
||||
}
|
||||
// err in call to os.Stat
|
||||
if err != nil {
|
||||
return pathType, err
|
||||
}
|
||||
|
||||
// checks whether the mode is the target mode
|
||||
isSpecificMode := func(mode, targetMode os.FileMode) bool {
|
||||
return mode&targetMode == targetMode
|
||||
}
|
||||
|
||||
mode := info.Mode()
|
||||
if mode.IsDir() {
|
||||
return FileTypeDirectory, nil
|
||||
} else if mode.IsRegular() {
|
||||
return FileTypeFile, nil
|
||||
} else if isSpecificMode(mode, os.ModeSocket) {
|
||||
return FileTypeSocket, nil
|
||||
} else if isSpecificMode(mode, os.ModeDevice) {
|
||||
if isSpecificMode(mode, os.ModeCharDevice) {
|
||||
return FileTypeCharDev, nil
|
||||
}
|
||||
return FileTypeBlockDev, nil
|
||||
}
|
||||
|
||||
return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,162 @@
|
||||
// +build !linux,!windows,!providerless
|
||||
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Mounter provides the default unsupported implementation of mount.Interface
|
||||
type Mounter struct {
|
||||
mounterPath string
|
||||
}
|
||||
|
||||
var errUnsupported = errors.New(" util/mount on this platform is not supported")
|
||||
|
||||
// New returns a mount.Interface for the current system.
|
||||
// It provides options to override the default mounter behavior.
|
||||
// mounterPath allows using an alternative to `/bin/mount` for mounting.
|
||||
func New(mounterPath string) Interface {
|
||||
return &Mounter{
|
||||
mounterPath: mounterPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Mount returns an error
|
||||
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// Unmount returns an error
|
||||
func (mounter *Mounter) Unmount(target string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// List returns an error
|
||||
func (mounter *Mounter) List() ([]MntPoint, error) {
|
||||
return []MntPoint{}, errUnsupported
|
||||
}
|
||||
|
||||
// IsMountPointMatch returns an error
|
||||
func (mounter *Mounter) IsMountPointMatch(mp MntPoint, dir string) bool {
|
||||
return (mp.Path == dir)
|
||||
}
|
||||
|
||||
// IsNotMountPoint returns an error
|
||||
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
|
||||
return IsNotMountPoint(mounter, dir)
|
||||
}
|
||||
|
||||
// IsLikelyNotMountPoint returns an error
|
||||
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
|
||||
return true, errUnsupported
|
||||
}
|
||||
|
||||
// GetDeviceNameFromMount returns an error
|
||||
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
|
||||
return "", errUnsupported
|
||||
}
|
||||
|
||||
func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
|
||||
return "", errUnsupported
|
||||
}
|
||||
|
||||
// DeviceOpened returns an error
|
||||
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
|
||||
return false, errUnsupported
|
||||
}
|
||||
|
||||
// PathIsDevice returns an error
|
||||
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
|
||||
return true, errUnsupported
|
||||
}
|
||||
|
||||
// MakeRShared returns an error
|
||||
func (mounter *Mounter) MakeRShared(path string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
|
||||
return mounter.Interface.Mount(source, target, fstype, options)
|
||||
}
|
||||
|
||||
func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
|
||||
return true, errUnsupported
|
||||
}
|
||||
|
||||
// GetFileType returns an error
|
||||
func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
|
||||
return FileType("fake"), errUnsupported
|
||||
}
|
||||
|
||||
// MakeDir returns an error
|
||||
func (mounter *Mounter) MakeDir(pathname string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// MakeFile returns an error
|
||||
func (mounter *Mounter) MakeFile(pathname string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// ExistsPath returns an error
|
||||
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
|
||||
return true, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// EvalHostSymlinks returns an error
|
||||
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
|
||||
return "", errUnsupported
|
||||
}
|
||||
|
||||
// PrepareSafeSubpath returns an error
|
||||
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
|
||||
return subPath.Path, nil, errUnsupported
|
||||
}
|
||||
|
||||
// CleanSubPaths returns an error
|
||||
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// SafeMakeDir returns an error
|
||||
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
|
||||
return errUnsupported
|
||||
}
|
||||
|
||||
// GetMountRefs returns an error
|
||||
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// GetFSGroup returns an error
|
||||
func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
|
||||
return -1, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// GetSELinuxSupport returns an error
|
||||
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
|
||||
return false, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// GetMode returns an error
|
||||
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
|
||||
return 0, errors.New("not implemented")
|
||||
}
|
||||
@ -0,0 +1,613 @@
|
||||
// +build windows,!providerless
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"k8s.io/klog"
|
||||
|
||||
utilpath "k8s.io/utils/path"
|
||||
)
|
||||
|
||||
// Mounter provides the default implementation of mount.Interface
|
||||
// for the windows platform. This implementation assumes that the
|
||||
// kubelet is running in the host's root mount namespace.
|
||||
type Mounter struct {
|
||||
mounterPath string
|
||||
}
|
||||
|
||||
// New returns a mount.Interface for the current system.
|
||||
// It provides options to override the default mounter behavior.
|
||||
// mounterPath allows using an alternative to `/bin/mount` for mounting.
|
||||
func New(mounterPath string) Interface {
|
||||
return &Mounter{
|
||||
mounterPath: mounterPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Mount : mounts source to target as NTFS with given options.
|
||||
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
|
||||
target = normalizeWindowsPath(target)
|
||||
|
||||
if source == "tmpfs" {
|
||||
klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options)
|
||||
return os.MkdirAll(target, 0755)
|
||||
}
|
||||
|
||||
parentDir := filepath.Dir(target)
|
||||
if err := os.MkdirAll(parentDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
|
||||
options, source, target, fstype)
|
||||
bindSource := ""
|
||||
|
||||
// tell it's going to mount azure disk or azure file according to options
|
||||
if bind, _, _ := isBind(options); bind {
|
||||
// mount azure disk
|
||||
bindSource = normalizeWindowsPath(source)
|
||||
} else {
|
||||
if len(options) < 2 {
|
||||
klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
|
||||
options, len(options), source, target)
|
||||
return nil
|
||||
}
|
||||
|
||||
// currently only cifs mount is supported
|
||||
if strings.ToLower(fstype) != "cifs" {
|
||||
return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
|
||||
}
|
||||
|
||||
bindSource = source
|
||||
|
||||
// use PowerShell Environment Variables to store user input string to prevent command line injection
|
||||
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
|
||||
cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
|
||||
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
|
||||
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`)
|
||||
|
||||
cmd := exec.Command("powershell", "/c", cmdLine)
|
||||
cmd.Env = append(os.Environ(),
|
||||
fmt.Sprintf("smbuser=%s", options[0]),
|
||||
fmt.Sprintf("smbpassword=%s", options[1]),
|
||||
fmt.Sprintf("smbremotepath=%s", source))
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output))
|
||||
}
|
||||
}
|
||||
|
||||
if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
|
||||
klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmount unmounts the target.
|
||||
func (mounter *Mounter) Unmount(target string) error {
|
||||
klog.V(4).Infof("azureMount: Unmount target (%q)", target)
|
||||
target = normalizeWindowsPath(target)
|
||||
if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
|
||||
klog.Errorf("rmdir failed: %v, output: %q", err, string(output))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns a list of all mounted filesystems. todo
|
||||
func (mounter *Mounter) List() ([]MntPoint, error) {
|
||||
return []MntPoint{}, nil
|
||||
}
|
||||
|
||||
// IsMountPointMatch determines if the mountpoint matches the dir
|
||||
func (mounter *Mounter) IsMountPointMatch(mp MntPoint, dir string) bool {
|
||||
return mp.Path == dir
|
||||
}
|
||||
|
||||
// IsNotMountPoint determines if a directory is a mountpoint.
|
||||
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
|
||||
return IsNotMountPoint(mounter, dir)
|
||||
}
|
||||
|
||||
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
|
||||
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
|
||||
stat, err := os.Lstat(file)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
// If current file is a symlink, then it is a mountpoint.
|
||||
if stat.Mode()&os.ModeSymlink != 0 {
|
||||
target, err := os.Readlink(file)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("readlink error: %v", err)
|
||||
}
|
||||
exists, err := mounter.ExistsPath(target)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
return !exists, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GetDeviceNameFromMount given a mnt point, find the device
|
||||
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
|
||||
return getDeviceNameFromMount(mounter, mountPath, pluginDir)
|
||||
}
|
||||
|
||||
// getDeviceNameFromMount find the device(drive) name in which
|
||||
// the mount path reference should match the given plugin directory. In case no mount path reference
|
||||
// matches, returns the volume name taken from its given mountPath
|
||||
func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
|
||||
refs, err := mounter.GetMountRefs(mountPath)
|
||||
if err != nil {
|
||||
klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
|
||||
return "", err
|
||||
}
|
||||
if len(refs) == 0 {
|
||||
return "", fmt.Errorf("directory %s is not mounted", mountPath)
|
||||
}
|
||||
basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath))
|
||||
for _, ref := range refs {
|
||||
if strings.Contains(ref, basemountPath) {
|
||||
volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
|
||||
return "", err
|
||||
}
|
||||
return volumeID, nil
|
||||
}
|
||||
}
|
||||
|
||||
return path.Base(mountPath), nil
|
||||
}
|
||||
|
||||
// DeviceOpened determines if the device is in use elsewhere
|
||||
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// PathIsDevice determines if a path is a device.
|
||||
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// MakeRShared checks that given path is on a mount with 'rshared' mount
|
||||
// propagation. Empty implementation here.
|
||||
func (mounter *Mounter) MakeRShared(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileType checks for sockets/block/character devices
|
||||
func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
|
||||
return getFileType(pathname)
|
||||
}
|
||||
|
||||
// MakeDir creates a new directory
|
||||
func (mounter *Mounter) MakeDir(pathname string) error {
|
||||
err := os.MkdirAll(pathname, os.FileMode(0755))
|
||||
if err != nil {
|
||||
if !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MakeFile creates an empty file
|
||||
func (mounter *Mounter) MakeFile(pathname string) error {
|
||||
f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
if !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExistsPath checks whether the path exists
|
||||
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
|
||||
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
|
||||
}
|
||||
|
||||
// EvalHostSymlinks returns the path name after evaluating symlinks
|
||||
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
|
||||
return filepath.EvalSymlinks(pathname)
|
||||
}
|
||||
|
||||
// check whether hostPath is within volume path
|
||||
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
|
||||
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
|
||||
if len(volumePath) == 0 || len(hostPath) == 0 {
|
||||
return []uintptr{}, nil
|
||||
}
|
||||
|
||||
finalSubPath, err := filepath.EvalSymlinks(hostPath)
|
||||
if err != nil {
|
||||
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
|
||||
}
|
||||
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
|
||||
if err != nil {
|
||||
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
|
||||
}
|
||||
|
||||
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
|
||||
}
|
||||
|
||||
// lock all intermediate subPath directories and check they are all within volumePath
|
||||
// volumePath & subPath should not contain any symlink, otherwise it will return error
|
||||
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
|
||||
if len(volumePath) == 0 || len(subPath) == 0 {
|
||||
return []uintptr{}, nil
|
||||
}
|
||||
|
||||
// get relative path to volumePath
|
||||
relSubPath, err := filepath.Rel(volumePath, subPath)
|
||||
if err != nil {
|
||||
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
|
||||
}
|
||||
if startsWithBackstep(relSubPath) {
|
||||
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
|
||||
}
|
||||
|
||||
if relSubPath == "." {
|
||||
// volumePath and subPath are equal
|
||||
return []uintptr{}, nil
|
||||
}
|
||||
|
||||
fileHandles := []uintptr{}
|
||||
var errorResult error
|
||||
|
||||
currentFullPath := volumePath
|
||||
dirs := strings.Split(relSubPath, string(os.PathSeparator))
|
||||
for _, dir := range dirs {
|
||||
// lock intermediate subPath directory first
|
||||
currentFullPath = filepath.Join(currentFullPath, dir)
|
||||
handle, err := lockPath(currentFullPath)
|
||||
if err != nil {
|
||||
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
|
||||
break
|
||||
}
|
||||
fileHandles = append(fileHandles, handle)
|
||||
|
||||
// make sure intermediate subPath directory does not contain symlink any more
|
||||
stat, err := os.Lstat(currentFullPath)
|
||||
if err != nil {
|
||||
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
|
||||
break
|
||||
}
|
||||
if stat.Mode()&os.ModeSymlink != 0 {
|
||||
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
|
||||
break
|
||||
}
|
||||
|
||||
if !PathWithinBase(currentFullPath, volumePath) {
|
||||
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return fileHandles, errorResult
|
||||
}
|
||||
|
||||
// unlockPath unlock directories
|
||||
func unlockPath(fileHandles []uintptr) {
|
||||
if fileHandles != nil {
|
||||
for _, handle := range fileHandles {
|
||||
syscall.CloseHandle(syscall.Handle(handle))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
|
||||
func lockPath(path string) (uintptr, error) {
|
||||
if len(path) == 0 {
|
||||
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
|
||||
}
|
||||
pathp, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return uintptr(syscall.InvalidHandle), err
|
||||
}
|
||||
access := uint32(syscall.GENERIC_READ)
|
||||
sharemode := uint32(syscall.FILE_SHARE_READ)
|
||||
createmode := uint32(syscall.OPEN_EXISTING)
|
||||
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
|
||||
return uintptr(fd), err
|
||||
}
|
||||
|
||||
// PrepareSafeSubpath locks all directories in subPath and check they're not symlinks.
|
||||
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
|
||||
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
|
||||
|
||||
// Unlock the directories when the container starts
|
||||
cleanupAction = func() {
|
||||
unlockPath(handles)
|
||||
}
|
||||
return subPath.Path, cleanupAction, err
|
||||
}
|
||||
|
||||
// CleanSubPaths no bind-mounts for subpaths are necessary on Windows
|
||||
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
|
||||
// Try to mount the disk
|
||||
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
|
||||
|
||||
if err := ValidateDiskNumber(source); err != nil {
|
||||
klog.Errorf("diskMount: formatAndMount failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if len(fstype) == 0 {
|
||||
// Use 'NTFS' as the default
|
||||
fstype = "NTFS"
|
||||
}
|
||||
|
||||
// format disk if it is unformatted(raw)
|
||||
cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
|
||||
" | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
|
||||
if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil {
|
||||
return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
|
||||
}
|
||||
klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)
|
||||
|
||||
driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
driverPath := driveLetter + ":"
|
||||
target = normalizeWindowsPath(target)
|
||||
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
|
||||
if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil {
|
||||
klog.Errorf("mklink failed: %v, output: %q", err, string(output))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeWindowsPath(path string) string {
|
||||
normalizedPath := strings.Replace(path, "/", "\\", -1)
|
||||
if strings.HasPrefix(normalizedPath, "\\") {
|
||||
normalizedPath = "c:" + normalizedPath
|
||||
}
|
||||
return normalizedPath
|
||||
}
|
||||
|
||||
// ValidateDiskNumber : disk number should be a number in [0, 99]
|
||||
func ValidateDiskNumber(disk string) error {
|
||||
diskNum, err := strconv.Atoi(disk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err)
|
||||
}
|
||||
|
||||
if diskNum < 0 || diskNum > 99 {
|
||||
return fmt.Errorf("disk number out of range: %q", disk)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get drive letter according to windows disk number
|
||||
func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) {
|
||||
cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum)
|
||||
output, err := exec.Run("powershell", "/c", cmd)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output))
|
||||
}
|
||||
if len(string(output)) < 1 {
|
||||
return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty")
|
||||
}
|
||||
return string(output)[:1], nil
|
||||
}
|
||||
|
||||
// getAllParentLinks walks all symbolic links and return all the parent targets recursively
|
||||
func getAllParentLinks(path string) ([]string, error) {
|
||||
const maxIter = 255
|
||||
links := []string{}
|
||||
for {
|
||||
links = append(links, path)
|
||||
if len(links) > maxIter {
|
||||
return links, fmt.Errorf("unexpected length of parent links: %v", links)
|
||||
}
|
||||
|
||||
fi, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return links, fmt.Errorf("Lstat: %v", err)
|
||||
}
|
||||
if fi.Mode()&os.ModeSymlink == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
path, err = os.Readlink(path)
|
||||
if err != nil {
|
||||
return links, fmt.Errorf("Readlink error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return links, nil
|
||||
}
|
||||
|
||||
// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
|
||||
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
|
||||
if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) {
|
||||
return []string{}, nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{pathname}, nil
|
||||
}
|
||||
|
||||
// GetFSGroup always returns 0. We actually don't set FSGroup on
|
||||
// windows platform, see SetVolumeOwnership implementation.
|
||||
func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// GetSELinuxSupport returns false
|
||||
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
|
||||
// Windows does not support SELinux.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// GetMode returns the filesystem mode.
|
||||
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
|
||||
info, err := os.Stat(pathname)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Mode(), nil
|
||||
}
|
||||
|
||||
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
|
||||
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
|
||||
realBase, err := filepath.EvalSymlinks(base)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
|
||||
}
|
||||
|
||||
realFullPath := filepath.Join(realBase, subdir)
|
||||
return doSafeMakeDir(realFullPath, realBase, perm)
|
||||
}
|
||||
|
||||
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
|
||||
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
|
||||
|
||||
if !PathWithinBase(pathname, base) {
|
||||
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
|
||||
}
|
||||
|
||||
// Quick check if the directory already exists
|
||||
s, err := os.Stat(pathname)
|
||||
if err == nil {
|
||||
// Path exists
|
||||
if s.IsDir() {
|
||||
// The directory already exists. It can be outside of the parent,
|
||||
// but there is no race-proof check.
|
||||
klog.V(4).Infof("Directory %s already exists", pathname)
|
||||
return nil
|
||||
}
|
||||
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
|
||||
}
|
||||
|
||||
// Find all existing directories
|
||||
existingPath, toCreate, err := findExistingPrefix(base, pathname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening directory %s: %s", pathname, err)
|
||||
}
|
||||
if len(toCreate) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure the existing directory is inside allowed base
|
||||
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
|
||||
}
|
||||
fullBasePath, err := filepath.EvalSymlinks(base)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read link %s: %s", base, err)
|
||||
}
|
||||
if !PathWithinBase(fullExistingPath, fullBasePath) {
|
||||
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
|
||||
}
|
||||
|
||||
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
|
||||
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
|
||||
defer unlockPath(fileHandles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
|
||||
currentPath := fullExistingPath
|
||||
// create the directories one by one, making sure nobody can change
|
||||
// created directory into symlink by lock that directory immediately
|
||||
for _, dir := range toCreate {
|
||||
currentPath = filepath.Join(currentPath, dir)
|
||||
klog.V(4).Infof("Creating %s", dir)
|
||||
if err := os.Mkdir(currentPath, perm); err != nil {
|
||||
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
|
||||
}
|
||||
handle, err := lockPath(currentPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
|
||||
}
|
||||
defer syscall.CloseHandle(syscall.Handle(handle))
|
||||
// make sure newly created directory does not contain symlink after lock
|
||||
stat, err := os.Lstat(currentPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
|
||||
}
|
||||
if stat.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findExistingPrefix finds prefix of pathname that exists. In addition, it
|
||||
// returns list of remaining directories that don't exist yet.
|
||||
func findExistingPrefix(base, pathname string) (string, []string, error) {
|
||||
rel, err := filepath.Rel(base, pathname)
|
||||
if err != nil {
|
||||
return base, nil, err
|
||||
}
|
||||
|
||||
if startsWithBackstep(rel) {
|
||||
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
|
||||
}
|
||||
|
||||
if rel == "." {
|
||||
// base and pathname are equal
|
||||
return pathname, []string{}, nil
|
||||
}
|
||||
|
||||
dirs := strings.Split(rel, string(filepath.Separator))
|
||||
|
||||
parent := base
|
||||
currentPath := base
|
||||
for i, dir := range dirs {
|
||||
parent = currentPath
|
||||
currentPath = filepath.Join(parent, dir)
|
||||
if _, err := os.Lstat(currentPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return parent, dirs[i:], nil
|
||||
}
|
||||
return base, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return pathname, []string{}, nil
|
||||
}
|
||||
2
vendor/modules.txt
vendored
2
vendor/modules.txt
vendored
@ -1831,6 +1831,8 @@ k8s.io/legacy-cloud-providers/aws
|
||||
k8s.io/legacy-cloud-providers/azure
|
||||
k8s.io/legacy-cloud-providers/azure/auth
|
||||
k8s.io/legacy-cloud-providers/gce
|
||||
k8s.io/legacy-cloud-providers/openstack
|
||||
k8s.io/legacy-cloud-providers/openstack/util/mount
|
||||
k8s.io/legacy-cloud-providers/vsphere
|
||||
k8s.io/legacy-cloud-providers/vsphere/testing
|
||||
k8s.io/legacy-cloud-providers/vsphere/vclib
|
||||
|
||||
Loading…
Reference in New Issue
Block a user