github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 15:25:57 +00:00
Code Issues Projects Releases Wiki Activity

Inital Quobyte dynamic provision

Browse Source
This commit is contained in:
Johannes Scheuermann 2016-08-20 15:39:30 +02:00
parent 00a203df1e
commit 0b7cb5f2ae
9 changed files with 458 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kube-controller-manager/app/plugins.go
View File

@ -45,6 +45,7 @@ import (
"k8s.io/kubernetes/pkg/volume/glusterfs"
"k8s.io/kubernetes/pkg/volume/host_path"
"k8s.io/kubernetes/pkg/volume/nfs"
"k8s.io/kubernetes/pkg/volume/quobyte"
"k8s.io/kubernetes/pkg/volume/rbd"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
)
@ -105,6 +106,8 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
// add rbd provisioner
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
if cloud != nil {
switch {
case aws.ProviderName == cloud.ProviderName():

88
examples/experimental/persistent-volume-provisioning/README.md
View File

@ -147,6 +147,94 @@ parameters:
* `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`.
* `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required.
#### Quobyte
<!-- BEGIN MUNGE: EXAMPLE quobyte/quobyte-storage-class.yaml -->
```yaml
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: slow
provisioner: kubernetes.io/quobyte
parameters:
quobyteApiServer: "http://138.68.74.142:7860"
registry: "138.68.74.142:7861"
adminSecretName: "quobyte-admin-secret"
adminSecretNamespace: "kube-system"
user: "root"
group: "root"
quobyteConfig: "BASE"
quobyteTenant: "DEFAULT"
```
[Download example](quobyte/quobyte-storage-class.yaml?raw=true)
<!-- END MUNGE: EXAMPLE quobyte/quobyte-storage-class.yaml -->
* **quobyteApiServer** API Server of Quobyte in the format http(s)://api-server:7860
* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a comma between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
* **adminSecretName** secret that holds information about the Quobyte user and the password to authenticate agains the API server.
* **adminSecretNamespace** The namespace for **adminSecretName**. Default is `default`.
* **user** maps all access to this user. Default is `root`.
* **group** maps all access to this group. Default is `nfsnobody`.
* **quobyteConfig** use the specified configuration to create the volume. You can create a new configuration or modify an existing one with the Web console or the quobyte CLI. Default is `BASE`
* **quobyteTenant** use the specified tenant to create/delete the volume. This Quobyte tenant has to be already present in Quobyte. Default is `DEFAULT`
First create Quobyte admin's Secret in the system namespace. Here the Secret is created in `kube-system`:
```
$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml --namespace=kube-system
```
Then create the Quobyte storage class:
```
$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml
```
Now create a PVC
```
$ kubectl create -f examples/experimental/persistent-volume-provisioning/claim1.json
```
Check the created PVC:
```
$ kubectl describe pvc
Name: claim1
Namespace: default
Status: Bound
Volume: pvc-bdb82652-694a-11e6-b811-080027242396
Labels: <none>
Capacity: 3Gi
Access Modes: RWO
No events.
$ kubectl describe pv
Name: pvc-bdb82652-694a-11e6-b811-080027242396
Labels: <none>
Status: Bound
Claim: default/claim1
Reclaim Policy: Delete
Access Modes: RWO
Capacity: 3Gi
Message:
Source:
Type: Quobyte (a Quobyte mount on the host that shares a pod's lifetime)
Registry: 138.68.79.14:7861
Volume: kubernetes-dynamic-pvc-bdb97c58-694a-11e6-91b6-080027242396
ReadOnly: false
No events.
```
Create a Pod to use the PVC:
```
$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/example-pod.yaml
```
### User provisioning requests
Users request dynamically provisioned storage by including a storage class in their `PersistentVolumeClaim`.

23
examples/experimental/persistent-volume-provisioning/quobyte/example-pod.yaml Normal file
View File

@ -0,0 +1,23 @@
apiVersion: v1
kind: ReplicationController
metadata:
name: server
spec:
replicas: 1
selector:
role: server
template:
metadata:
labels:
role: server
spec:
containers:
- name: server
image: nginx
volumeMounts:
- mountPath: /var/lib/www/html
name: quobytepvc
volumes:
- name: quobytepvc
persistentVolumeClaim:
claimName: claim1

7
examples/experimental/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: Secret
metadata:
name: quobyte-admin-secret
data:
password: cXVvYnl0ZQ==
user: YWRtaW4=

14
examples/experimental/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml Normal file
View File

@ -0,0 +1,14 @@
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: slow
provisioner: kubernetes.io/quobyte
parameters:
quobyteApiServer: "http://138.68.74.142:7860"
registry: "138.68.74.142:7861"
adminSecretName: "quobyte-admin-secret"
adminSecretNamespace: "kube-system"
user: "root"
group: "root"
quobyteConfig: "BASE"
quobyteTenant: "DEFAULT"

6
examples/volumes/quobyte/Readme.md
View File

@ -94,11 +94,11 @@ spec:
<!-- END MUNGE: EXAMPLE ./quobyte-pod.yaml -->
Parameters:
* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a semicolon between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a comma between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
* **volume** volume represents a Quobyte volume which must be created before usage.
* **readOnly** is the boolean that sets the mountpoint readOnly or readWrite.
* **user** maps all access to this user. Default is root.
* **group** maps all access to this group. Default is empty.
* **user** maps all access to this user. Default is `root`.
* **group** maps all access to this group. Default is `nfsnobody`.
Creating the pod:

243
pkg/volume/quobyte/quobyte.go
View File

@ -20,14 +20,18 @@ import (
"fmt"
"os"
"path"
goStrings "strings"
"github.com/golang/glog"
"github.com/pborman/uuid"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/types"
"k8s.io/kubernetes/pkg/util/exec"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/util/strings"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
)
// ProbeVolumePlugins is the primary entrypoint for volume plugins.
@ -39,11 +43,27 @@ type quobytePlugin struct {
host volume.VolumeHost
}
// This user is used to authenticate against the
// Quobyte API server and holds all information
type quobyteAPIConfig struct {
quobyteUser string
quobytePassword string
quobyteApiServer string
}
var _ volume.VolumePlugin = &quobytePlugin{}
var _ volume.PersistentVolumePlugin = &quobytePlugin{}
var _ volume.DeletableVolumePlugin = &quobytePlugin{}
var _ volume.ProvisionableVolumePlugin = &quobytePlugin{}
var _ volume.Provisioner = &quobyteVolumeProvisioner{}
var _ volume.Deleter = &quobyteVolumeDeleter{}
const (
quobytePluginName = "kubernetes.io/quobyte"
annotationQuobyteAPIServer = "quobyte.kubernetes.io/api"
annotationQuobyteAPISecret = "quobyte.kubernetes.io/apiuser"
annotationQuobyteAPISecretNamespace = "quobyte.kubernetes.io/apipassword"
)
func (plugin *quobytePlugin) Init(host volume.VolumeHost) error {
@ -149,7 +169,8 @@ func (plugin *quobytePlugin) newMounterInternal(spec *volume.Spec, pod *api.Pod,
plugin: plugin,
},
registry: source.Registry,
readOnly: readOnly}, nil
readOnly: readOnly,
}, nil
}
func (plugin *quobytePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
@ -157,21 +178,26 @@ func (plugin *quobytePlugin) NewUnmounter(volName string, podUID types.UID) (vol
}
func (plugin *quobytePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
return &quobyteUnmounter{&quobyte{
volName: volName,
mounter: mounter,
pod: &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
plugin: plugin,
}}, nil
return &quobyteUnmounter{
&quobyte{
volName: volName,
mounter: mounter,
pod: &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
plugin: plugin,
},
}, nil
}
// Quobyte volumes represent a bare host directory mount of an quobyte export.
//TODO add configuration + tenant
type quobyte struct {
volName string
pod *api.Pod
user string
group string
volume string
tenant string
config string
mounter mount.Interface
plugin *quobytePlugin
volume.MetricsNil
@ -226,22 +252,22 @@ func (mounter *quobyteMounter) SetUpAt(dir string, fsGroup *int64) error {
}
// GetPath returns the path to the user specific mount of a Quobyte volume
// Returns a path in the format ../user@volume e.g. ../root@MyVolume
// or if a group is set ../user#group@volume
// Returns a path in the format ../user#group@volume
func (quobyteVolume *quobyte) GetPath() string {
user := quobyteVolume.user
if len(user) == 0 {
user = "root"
}
group := quobyteVolume.group
if len(group) == 0 {
group = "nfsnobody"
}
// Quobyte has only one mount in the PluginDir where all Volumes are mounted
// The Quobyte client does a fixed-user mapping
pluginDir := quobyteVolume.plugin.host.GetPluginDir(strings.EscapeQualifiedNameForDisk(quobytePluginName))
if len(quobyteVolume.group) > 0 {
return path.Join(pluginDir, fmt.Sprintf("%s#%s@%s", user, quobyteVolume.group, quobyteVolume.volume))
}
return path.Join(pluginDir, fmt.Sprintf("%s@%s", user, quobyteVolume.volume))
return path.Join(pluginDir, fmt.Sprintf("%s#%s@%s", user, group, quobyteVolume.volume))
}
type quobyteUnmounter struct {
@ -258,3 +284,192 @@ func (unmounter *quobyteUnmounter) TearDown() error {
func (unmounter *quobyteUnmounter) TearDownAt(dir string) error {
return nil
}
type quobyteVolumeDeleter struct {
*quobyteMounter
pv *api.PersistentVolume
}
func (plugin *quobytePlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Quobyte == nil {
return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.Quobyte is nil")
}
return plugin.newDeleterInternal(spec)
}
func (plugin *quobytePlugin) newDeleterInternal(spec *volume.Spec) (volume.Deleter, error) {
source, readOnly, err := getVolumeSource(spec)
if err != nil {
return nil, err
}
return &quobyteVolumeDeleter{
quobyteMounter: &quobyteMounter{
quobyte: &quobyte{
volName: spec.Name(),
user: source.User,
group: source.Group,
volume: source.Volume,
plugin: plugin,
},
registry: source.Registry,
readOnly: readOnly,
},
pv: spec.PersistentVolume,
}, nil
}
func (plugin *quobytePlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
if len(options.AccessModes) == 0 {
options.AccessModes = plugin.GetAccessModes()
}
return plugin.newProvisionerInternal(options)
}
func (plugin *quobytePlugin) newProvisionerInternal(options volume.VolumeOptions) (volume.Provisioner, error) {
return &quobyteVolumeProvisioner{
quobyteMounter: &quobyteMounter{
quobyte: &quobyte{
plugin: plugin,
},
},
options: options,
}, nil
}
type quobyteVolumeProvisioner struct {
*quobyteMounter
options volume.VolumeOptions
}
func (provisioner *quobyteVolumeProvisioner) Provision() (*api.PersistentVolume, error) {
if provisioner.options.Selector != nil {
return nil, fmt.Errorf("claim Selector is not supported")
}
var apiServer, adminSecretName, quobyteUser, quobytePassword string
adminSecretNamespace := "default"
provisioner.config = "BASE"
provisioner.tenant = "DEFAULT"
for k, v := range provisioner.options.Parameters {
switch goStrings.ToLower(k) {
case "registry":
provisioner.registry = v
case "adminsecretname":
adminSecretName = v
case "adminsecretnamespace":
adminSecretNamespace = v
case "quobyteapiserver":
apiServer = v
case "user":
provisioner.user = v
case "group":
provisioner.group = v
case "quobytetenant":
provisioner.tenant = v
case "quobyteconfig":
provisioner.config = v
default:
return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, provisioner.plugin.GetPluginName())
}
}
secretMap, err := util.GetSecret(adminSecretNamespace, adminSecretName, provisioner.plugin.host.GetKubeClient())
if err != nil {
return nil, err
}
var ok bool
if quobyteUser, ok = secretMap["user"]; !ok {
return nil, fmt.Errorf("Missing \"user\" in secret")
}
if quobytePassword, ok = secretMap["password"]; !ok {
return nil, fmt.Errorf("Missing \"password\" in secret")
}
if !validateRegistry(provisioner.registry) {
return nil, fmt.Errorf("Quoybte registry missing or malformed: must be a host:port pair or multiple pairs seperated by commas")
}
if len(apiServer) == 0 {
return nil, fmt.Errorf("Quoybte API server missing or malformed: must be a http(s)://host:port pair or multiple pairs seperated by commas")
}
// create random image name
provisioner.volume = fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
cfg := &quobyteAPIConfig{
quobyteApiServer: apiServer,
quobyteUser: quobyteUser,
quobytePassword: quobytePassword,
}
manager := &quobyteVolumeManager{
config: cfg,
}
vol, sizeGB, err := manager.createVolume(provisioner)
if err != nil {
return nil, err
}
pv := new(api.PersistentVolume)
pv.Spec.PersistentVolumeSource.Quobyte = vol
pv.Spec.PersistentVolumeReclaimPolicy = provisioner.options.PersistentVolumeReclaimPolicy
pv.Spec.AccessModes = provisioner.options.AccessModes
pv.Spec.Capacity = api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
}
util.AddVolumeAnnotations(pv, map[string]string{
annotationQuobyteAPIServer: apiServer,
annotationQuobyteAPISecret: adminSecretName,
annotationQuobyteAPISecretNamespace: adminSecretNamespace,
})
return pv, nil
}
func (deleter *quobyteVolumeDeleter) GetPath() string {
return deleter.quobyte.GetPath()
}
func (deleter *quobyteVolumeDeleter) Delete() error {
var quobyteUser, quobytePassword string
annotations, err := util.ParseVolumeAnnotations(deleter.pv, []string{
annotationQuobyteAPISecret,
annotationQuobyteAPISecretNamespace,
annotationQuobyteAPIServer})
if err != nil {
return err
}
secretMap, err := util.GetSecret(
annotations[annotationQuobyteAPISecretNamespace],
annotations[annotationQuobyteAPISecret],
deleter.plugin.host.GetKubeClient())
if err != nil {
return err
}
var ok bool
if quobyteUser, ok = secretMap["user"]; !ok {
return fmt.Errorf("Missing \"user\" in secret")
}
if quobytePassword, ok = secretMap["password"]; !ok {
return fmt.Errorf("Missing \"password\" in secret")
}
manager := &quobyteVolumeManager{
config: &quobyteAPIConfig{
quobyteUser: quobyteUser,
quobytePassword: quobytePassword,
quobyteApiServer: annotations[annotationQuobyteAPIServer],
},
}
return manager.deleteVolume(deleter)
}

60
pkg/volume/quobyte/quobyte_util.go
View File

@ -17,12 +17,58 @@ limitations under the License.
package quobyte
import (
"net"
"path"
"strings"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/volume"
"github.com/golang/glog"
quobyte_api "github.com/quobyte/api"
)
type quobyteVolumeManager struct {
config *quobyteAPIConfig
}
func (manager *quobyteVolumeManager) createVolume(provisioner *quobyteVolumeProvisioner) (quobyte *api.QuobyteVolumeSource, size int, err error) {
volumeSize := int(volume.RoundUpSize(provisioner.options.Capacity.Value(), 1024*1024*1024))
// Quobyte has the concept of Volumes which doen't have a specific size (they can grow unlimited)
// to simulate a size constraint we could set here a Quota
volumeRequest := &quobyte_api.CreateVolumeRequest{
Name: provisioner.volume,
RootUserID: provisioner.user,
RootGroupID: provisioner.group,
TenantID: provisioner.tenant,
ConfigurationName: provisioner.config,
}
if _, err := manager.createQuobyteClient().CreateVolume(volumeRequest); err != nil {
return &api.QuobyteVolumeSource{}, volumeSize, err
}
glog.V(4).Infof("Created Quobyte volume %s", provisioner.volume)
return &api.QuobyteVolumeSource{
Registry: provisioner.registry,
Volume: provisioner.volume,
User: provisioner.user,
Group: provisioner.group,
}, volumeSize, nil
}
func (manager *quobyteVolumeManager) deleteVolume(deleter *quobyteVolumeDeleter) error {
return manager.createQuobyteClient().DeleteVolumeByName(deleter.volume, deleter.tenant)
}
func (manager *quobyteVolumeManager) createQuobyteClient() *quobyte_api.QuobyteClient {
return quobyte_api.NewQuobyteClient(
manager.config.quobyteApiServer,
manager.config.quobyteUser,
manager.config.quobytePassword,
)
}
func (mounter *quobyteMounter) pluginDirIsMounted(pluginDir string) (bool, error) {
mounts, err := mounter.mounter.List()
if err != nil {
@ -46,3 +92,17 @@ func (mounter *quobyteMounter) pluginDirIsMounted(pluginDir string) (bool, error
func (mounter *quobyteMounter) correctTraillingSlash(regStr string) string {
return path.Clean(regStr) + "/"
}
func validateRegistry(registry string) bool {
if len(registry) == 0 {
return false
}
for _, hostPortPair := range strings.Split(registry, ",") {
if _, _, err := net.SplitHostPort(hostPortPair); err != nil {
return false
}
}
return true
}

31
pkg/volume/util/util.go
View File

@ -22,6 +22,7 @@ import (
"path"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/util/mount"
)
@ -126,3 +127,33 @@ func GetSecret(namespace, secretName string, kubeClient clientset.Interface) (ma
}
return secret, nil
}
// AddVolumeAnnotations adds a golang Map as annotation to a PersistentVolume
func AddVolumeAnnotations(pv *api.PersistentVolume, annotations map[string]string) {
if pv.Annotations == nil {
pv.Annotations = map[string]string{}
}
for k, v := range annotations {
pv.Annotations[k] = v
}
}
// ParseVolumeAnnotations reads the defined annoations from a PersistentVolume
func ParseVolumeAnnotations(pv *api.PersistentVolume, parseAnnotations []string) (map[string]string, error) {
result := map[string]string{}
if pv.Annotations == nil {
return result, fmt.Errorf("cannot parse volume annotations: no annotations found")
}
for _, annotation := range parseAnnotations {
if val, ok := pv.Annotations[annotation]; ok {
result[annotation] = val
} else {
return result, fmt.Errorf("cannot parse volume annotations: annotation %s not found", annotation)
}
}
return result, nil
}