From d0182b4eb3d86065afce4817bca7a1b9fd7459fd Mon Sep 17 00:00:00 2001 From: "JEONGTAE.KIM@ORACLE.COM" Date: Tue, 24 Aug 2021 16:30:55 -0600 Subject: [PATCH 1/5] Initial publication of ZFSSA CSI driver --- CONTRIBUTING.md | 38 + INSTALLATION.md | 319 ++ LICENSE.txt | 35 + Makefile | 8 + README.md | 80 +- SECURITY.md | 17 + TEST.md | 139 + THIRD_PARTY_LICENSES.txt | 2579 +++++++++++++++++ VERSION | 1 + cmd/zfssa-csi-driver/main.go | 30 + deploy/helm/k8s-1.17/Chart.yaml | 4 + .../templates/00-zfssa-csi-secret.yaml | 9 + .../templates/01-zfssa-csi-certs.yaml | 8 + .../k8s-1.17/templates/02-zfssa-csi-rbac.yaml | 84 + .../templates/03-zfssa-csi-driver.yaml | 139 + .../templates/04-zfssa-csi-provisioner.yaml | 90 + deploy/helm/k8s-1.17/values.yaml | 42 + deploy/helm/local-values/README.md | 5 + ....storage.k8s.io_volumesnapshotclasses.yaml | 85 + ...storage.k8s.io_volumesnapshotcontents.yaml | 233 ++ ...apshot.storage.k8s.io_volumesnapshots.yaml | 188 ++ .../rbac-snapshot-controller.yaml | 80 + .../setup-snapshot-controller.yaml | 26 + examples/block-pv/Chart.yaml | 4 + examples/block-pv/README.md | 110 + .../block-pv/templates/00-storage-class.yaml | 20 + examples/block-pv/templates/01-pv.yaml | 24 + examples/block-pv/templates/02-pvc.yaml | 12 + examples/block-pv/templates/03-pod.yaml | 20 + examples/block-pv/values.yaml | 20 + .../block-pod-restored-volume.yaml | 21 + .../block-pvc-from-snapshot.yaml | 16 + examples/block-snapshot/block-snapshot.yaml | 8 + examples/block-vsc/Chart.yaml | 4 + examples/block-vsc/README.md | 193 ++ .../block-vsc/templates/00-storage-class.yaml | 20 + examples/block-vsc/templates/01-pvc.yaml | 12 + .../templates/02-volume-snapshot-class.yaml | 6 + examples/block-vsc/templates/03-pod.yaml | 20 + examples/block-vsc/values.yaml | 19 + examples/block/Chart.yaml | 4 + examples/block/README.md | 63 + .../block/templates/00-storage-class.yaml | 20 + examples/block/templates/01-pvc.yaml | 12 + examples/block/templates/02-pod.yaml | 20 + examples/block/values.yaml | 18 + examples/helm/sauron-storage/Chart.yaml | 4 + .../templates/00-sauron-sc.yaml | 20 + .../templates/01-sauron-pvc.yaml | 75 + examples/helm/sauron-storage/values.yaml | 21 + examples/local-values/README.md | 6 + examples/nfs-exp/Chart.yaml | 4 + examples/nfs-exp/README.md | 174 ++ .../nfs-exp/templates/00-storage-class.yaml | 21 + examples/nfs-exp/templates/01-pvc.yaml | 12 + examples/nfs-exp/templates/02-pod.yaml | 21 + examples/nfs-exp/values.yaml | 19 + examples/nfs-multi/Chart.yaml | 4 + examples/nfs-multi/README.md | 46 + .../nfs-multi/templates/00-storage-class.yaml | 20 + examples/nfs-multi/templates/01-pvc.yaml | 74 + examples/nfs-multi/templates/02-pod.yaml | 49 + examples/nfs-multi/values.yaml | 27 + examples/nfs-pv/Chart.yaml | 4 + examples/nfs-pv/README.md | 91 + .../nfs-pv/templates/00-storage-class.yaml | 20 + examples/nfs-pv/templates/01-pv.yaml | 28 + examples/nfs-pv/templates/02-pvc.yaml | 12 + examples/nfs-pv/templates/03-pod.yaml | 20 + examples/nfs-pv/values.yaml | 21 + .../nfs-snapshot/nfs-pod-restored-volume.yaml | 21 + .../nfs-snapshot/nfs-pvc-from-snapshot.yaml | 16 + examples/nfs-snapshot/nfs-snapshot.yaml | 8 + examples/nfs-vsc/Chart.yaml | 4 + examples/nfs-vsc/README.md | 193 ++ .../nfs-vsc/templates/00-storage-class.yaml | 20 + examples/nfs-vsc/templates/01-pvc.yaml | 12 + .../templates/02-volume-snapshot-class.yaml | 6 + examples/nfs-vsc/templates/03-pod.yaml | 21 + examples/nfs-vsc/values.yaml | 20 + examples/nfs/Chart.yaml | 4 + examples/nfs/README.md | 83 + examples/nfs/templates/00-storage-class.yaml | 20 + examples/nfs/templates/01-pvc.yaml | 12 + examples/nfs/templates/02-pod.yaml | 21 + examples/nfs/values.yaml | 19 + go.mod | 53 + pkg/service/cluster.go | 79 + pkg/service/controller.go | 688 +++++ pkg/service/controller_block.go | 381 +++ pkg/service/controller_fs.go | 355 +++ pkg/service/identity.go | 83 + pkg/service/iscsi.go | 468 +++ pkg/service/mount.go | 61 + pkg/service/node.go | 272 ++ pkg/service/node_block.go | 216 ++ pkg/service/node_fs.go | 101 + pkg/service/service.go | 393 +++ pkg/service/snapshots.go | 222 ++ pkg/service/volumes.go | 647 +++++ pkg/utils/log_utils.go | 128 + pkg/utils/sync.go | 42 + pkg/utils/volume_id.go | 279 ++ pkg/zfssarest/zfssa_fs.go | 203 ++ pkg/zfssarest/zfssa_lun.go | 201 ++ pkg/zfssarest/zfssa_pool.go | 85 + pkg/zfssarest/zfssa_project.go | 65 + pkg/zfssarest/zfssa_rest.go | 414 +++ pkg/zfssarest/zfssa_san.go | 48 + pkg/zfssarest/zfssa_schema.go | 71 + pkg/zfssarest/zfssa_snapshots.go | 146 + release-tools/build.make | 51 + 112 files changed, 11929 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md create mode 100644 INSTALLATION.md create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 SECURITY.md create mode 100644 TEST.md create mode 100644 THIRD_PARTY_LICENSES.txt create mode 100644 VERSION create mode 100644 cmd/zfssa-csi-driver/main.go create mode 100644 deploy/helm/k8s-1.17/Chart.yaml create mode 100644 deploy/helm/k8s-1.17/templates/00-zfssa-csi-secret.yaml create mode 100644 deploy/helm/k8s-1.17/templates/01-zfssa-csi-certs.yaml create mode 100644 deploy/helm/k8s-1.17/templates/02-zfssa-csi-rbac.yaml create mode 100644 deploy/helm/k8s-1.17/templates/03-zfssa-csi-driver.yaml create mode 100644 deploy/helm/k8s-1.17/templates/04-zfssa-csi-provisioner.yaml create mode 100644 deploy/helm/k8s-1.17/values.yaml create mode 100644 deploy/helm/local-values/README.md create mode 100644 deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml create mode 100644 deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml create mode 100644 deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshots.yaml create mode 100644 deploy/k8s-1.17/snapshot-controller/rbac-snapshot-controller.yaml create mode 100644 deploy/k8s-1.17/snapshot-controller/setup-snapshot-controller.yaml create mode 100644 examples/block-pv/Chart.yaml create mode 100644 examples/block-pv/README.md create mode 100644 examples/block-pv/templates/00-storage-class.yaml create mode 100644 examples/block-pv/templates/01-pv.yaml create mode 100644 examples/block-pv/templates/02-pvc.yaml create mode 100644 examples/block-pv/templates/03-pod.yaml create mode 100644 examples/block-pv/values.yaml create mode 100644 examples/block-snapshot/block-pod-restored-volume.yaml create mode 100644 examples/block-snapshot/block-pvc-from-snapshot.yaml create mode 100644 examples/block-snapshot/block-snapshot.yaml create mode 100644 examples/block-vsc/Chart.yaml create mode 100644 examples/block-vsc/README.md create mode 100644 examples/block-vsc/templates/00-storage-class.yaml create mode 100644 examples/block-vsc/templates/01-pvc.yaml create mode 100644 examples/block-vsc/templates/02-volume-snapshot-class.yaml create mode 100644 examples/block-vsc/templates/03-pod.yaml create mode 100644 examples/block-vsc/values.yaml create mode 100644 examples/block/Chart.yaml create mode 100644 examples/block/README.md create mode 100644 examples/block/templates/00-storage-class.yaml create mode 100644 examples/block/templates/01-pvc.yaml create mode 100644 examples/block/templates/02-pod.yaml create mode 100644 examples/block/values.yaml create mode 100644 examples/helm/sauron-storage/Chart.yaml create mode 100644 examples/helm/sauron-storage/templates/00-sauron-sc.yaml create mode 100644 examples/helm/sauron-storage/templates/01-sauron-pvc.yaml create mode 100644 examples/helm/sauron-storage/values.yaml create mode 100644 examples/local-values/README.md create mode 100644 examples/nfs-exp/Chart.yaml create mode 100644 examples/nfs-exp/README.md create mode 100644 examples/nfs-exp/templates/00-storage-class.yaml create mode 100644 examples/nfs-exp/templates/01-pvc.yaml create mode 100644 examples/nfs-exp/templates/02-pod.yaml create mode 100644 examples/nfs-exp/values.yaml create mode 100644 examples/nfs-multi/Chart.yaml create mode 100644 examples/nfs-multi/README.md create mode 100644 examples/nfs-multi/templates/00-storage-class.yaml create mode 100644 examples/nfs-multi/templates/01-pvc.yaml create mode 100644 examples/nfs-multi/templates/02-pod.yaml create mode 100644 examples/nfs-multi/values.yaml create mode 100644 examples/nfs-pv/Chart.yaml create mode 100644 examples/nfs-pv/README.md create mode 100644 examples/nfs-pv/templates/00-storage-class.yaml create mode 100644 examples/nfs-pv/templates/01-pv.yaml create mode 100644 examples/nfs-pv/templates/02-pvc.yaml create mode 100644 examples/nfs-pv/templates/03-pod.yaml create mode 100644 examples/nfs-pv/values.yaml create mode 100644 examples/nfs-snapshot/nfs-pod-restored-volume.yaml create mode 100644 examples/nfs-snapshot/nfs-pvc-from-snapshot.yaml create mode 100644 examples/nfs-snapshot/nfs-snapshot.yaml create mode 100644 examples/nfs-vsc/Chart.yaml create mode 100644 examples/nfs-vsc/README.md create mode 100644 examples/nfs-vsc/templates/00-storage-class.yaml create mode 100644 examples/nfs-vsc/templates/01-pvc.yaml create mode 100644 examples/nfs-vsc/templates/02-volume-snapshot-class.yaml create mode 100644 examples/nfs-vsc/templates/03-pod.yaml create mode 100644 examples/nfs-vsc/values.yaml create mode 100644 examples/nfs/Chart.yaml create mode 100644 examples/nfs/README.md create mode 100644 examples/nfs/templates/00-storage-class.yaml create mode 100644 examples/nfs/templates/01-pvc.yaml create mode 100644 examples/nfs/templates/02-pod.yaml create mode 100644 examples/nfs/values.yaml create mode 100644 go.mod create mode 100644 pkg/service/cluster.go create mode 100644 pkg/service/controller.go create mode 100644 pkg/service/controller_block.go create mode 100644 pkg/service/controller_fs.go create mode 100644 pkg/service/identity.go create mode 100644 pkg/service/iscsi.go create mode 100644 pkg/service/mount.go create mode 100644 pkg/service/node.go create mode 100644 pkg/service/node_block.go create mode 100644 pkg/service/node_fs.go create mode 100644 pkg/service/service.go create mode 100644 pkg/service/snapshots.go create mode 100644 pkg/service/volumes.go create mode 100644 pkg/utils/log_utils.go create mode 100644 pkg/utils/sync.go create mode 100644 pkg/utils/volume_id.go create mode 100644 pkg/zfssarest/zfssa_fs.go create mode 100644 pkg/zfssarest/zfssa_lun.go create mode 100644 pkg/zfssarest/zfssa_pool.go create mode 100644 pkg/zfssarest/zfssa_project.go create mode 100644 pkg/zfssarest/zfssa_rest.go create mode 100644 pkg/zfssarest/zfssa_san.go create mode 100644 pkg/zfssarest/zfssa_schema.go create mode 100644 pkg/zfssarest/zfssa_snapshots.go create mode 100644 release-tools/build.make diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1e6a4af --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +# Contributing + +Oracle welcomes contributions to this repository from anyone. If you want to submit a pull request to fix a bug or enhance code, please first open an issue and link to that issue when you submit your pull request. If you have any questions about a possible contribution, feel free to open an issue too. + +## Contributing to the zfssa-csi-driver repository + +Pull requests can be made under +[The Oracle Contributor Agreement](https://www.oracle.com/technetwork/community/oca-486395.html) (OCA). + +For pull requests to be accepted, the bottom of your commit message must have +the following line using your name and e-mail address as it appears in the +OCA Signatories list. + +``` +Signed-off-by: Your Name +``` + +This can be automatically added to pull requests by committing with: + +``` +git commit --signoff +``` + +Only pull requests from committers that can be verified as having +signed the OCA can be accepted. + +### Pull request process + +1. Fork this repository +1. Create a branch in your fork to implement the changes. We recommend using +the issue number as part of your branch name, e.g. `1234-fixes` +1. Ensure that any documentation is updated with the changes that are required +by your fix. +1. Ensure that any samples are updated if the base image has been changed. +1. Submit the pull request. *Do not leave the pull request blank*. Explain exactly +what your changes are meant to do and provide simple steps on how to validate +your changes. Ensure that you reference the issue you created as well. +We will assign the pull request to 2-3 people for review before it is merged. diff --git a/INSTALLATION.md b/INSTALLATION.md new file mode 100644 index 0000000..20c4604 --- /dev/null +++ b/INSTALLATION.md @@ -0,0 +1,319 @@ +# Installation of Oracle ZFS Storage Appliance CSI Plugin +This document reviews how to install and make use of the driver. + +## Requirements +Ensure the following information and requirements can be met prior to installation. + +* The following ZFS Storage Appliance information (see your ZFFSA device administrator): + + * The name or the IP address of the appliance. If it's a name, it must be DNS resolvable. + When the appliance is a clustered system, the connection for management operations is tied + to a head in driver deployment. You will see different driver behaviors in takeover/failback + scenarios with the target storage appliance depending on the management interface settings and + if it remains locked to the failed node or not. + * A login access to your ZFSSA in the form of a user login and associated password. + It is desirable to create a normal login user with required authorizations. + * The appliance certificate for the REST endpoint is available. + * The name of the appliance storage pool from which volumes will be provisioned. + * The name of the project in the pool. + * In secure mode, the driver supports only TLSv1.2 for HTTPS connection to ZFSSA. Make sure that + TLSv1.2 is enabled for HTTPS service on ZFSSA. + + The user on the appliance must have a minimum of the following authorizations (where pool and project are those + that will be used in the storage class), root should not be used for provisioning. + + * Object: nas...* + * Permissions: + * changeAccessProps + * changeGeneralProps + * changeProtocolProps + * changeSpaceProps + * changeUserQuota + * clearLocks + * clone + * createShare + * destroy + * rollback + * scheduleSnap + * takeSnap + * destroySnap + * renameSnap + + The File system being exported must have 'Share Mode' set to 'Read/Write' in the section 'NFS' of the tab 'Protocol' + of the file system (Under 'Shares'). + + More than one pool/project are possible if there are storage classes that identify different + pools and projects. + +* The Kubernetes cluster namespace you must use (see your cluster administrator) +* Sidecar images + + Make sure you have access to the registry or registries containing these images from the worker nodes. The image pull + policy (`imagePullPolicy`) is set to `IfNotPresent` in the deployment files. During the first deployment the + Container Runtime will likely try to pull them. If your Container Runtime cannot access the images you will have to + pull them manually before deployment. The required images are: + + * node-driver-registar v2.0.0+. + * external-attacher v3.0.2+. + * external-provisioner v2.0.5+. + * external-resizer v1.1.0+. + * external-snapshotter v3.0.3+. + + The common container images for those images are: + + * k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.0 + * k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 + * k8s.gcr.io/sig-storage/csi-provisioner:v2.0.5 + * k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 + * k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3 + +* Plugin image + + You can pull the plugin image from a registry that you know hosts it or you can generate it and store it in one of + your registries. In any case, as for the sidecar images, the Container Runtime must have access to that registry. + If not you will have to pull it manually before deployment. If you choose to generate the plugin yourself use + version 1.13.8 or above of the Go compiler. + +## Setup + +This volume driver supports both NFS (filesystem) and iSCSI (block) volumes. Preparation for iSCSI, at this time, will +take some setup, please see the information below. + +### iSCSI Environment + +Install iSCSI client utilities on the Kubernetes worker nodes: + +```bash +$ yum install iscsi-initiator-utils -y +``` + +Verify `iscsid` and `iscsi` are running after installation (systemctl status iscsid iscsi). + +* Create an initiator group on the Oracle ZFS Storage Appliance *per worker node name*. For example, if +your worker node name is `pmonday-olcne-worker-0`, then there should be an initiator group named `pmonday-olcne-worker-0` +on the target appliance with the IQN of the worker node. The initiator can be determined by looking at +`/etc/iscsi/initiatorname.iscsi`. +* Create one or more targets and target groups on the interface that you intend to use for iSCSI traffic. +* CHAP is not supported at this time. +* Cloud instances often have duplicate IQNs, these MUST be regenerated and unique or connection storms +happen ([Instructions](https://www.thegeekdiary.com/how-to-modify-the-iscsi-initiator-id-in-linux/)). +* There are cases where fresh instances do not start the iscsi service properly with the following, +modify the iscsi.service to remove the ConditionDirectoryNotEmpty temporarily +``` +Condition: start condition failed at Wed 2020-10-28 18:37:35 GMT; 1 day 4h ago + ConditionDirectoryNotEmpty=/var/lib/iscsi/nodes was not met +``` +* iSCSI may get timeouts in particular networking conditions. Review the following web pages for possible +solutions. The first involves +[modifying sysctl](https://www.thegeekdiary.com/iscsiadm-discovery-timeout-with-two-or-more-network-interfaces-in-centos-rhel/), +the second involves changing the +[replacement timeout](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/storage_administration_guide/iscsi-replacement-timeout) +for iSCSI. + +* There is a condition where a 'uefi' target creates noise in iscsi discovery, this is noticeable in the + iscsid output (systemctl status iscsid). This issue appears to be in Oracle Linux 7 in a virtualized + environment: +``` +● iscsid.service - Open-iSCSI + Loaded: loaded (/usr/lib/systemd/system/iscsid.service; disabled; vendor preset: disabled) + Active: active (running) since Wed 2020-10-28 17:30:17 GMT; 1 day 23h ago + Docs: man:iscsid(8) + man:iscsiuio(8) + man:iscsiadm(8) + Main PID: 1632 (iscsid) + Status: "Ready to process requests" + Tasks: 1 + Memory: 6.4M + CGroup: /system.slice/iscsid.service + └─1632 /sbin/iscsid -f -d2 + +Oct 30 16:23:02 pbm-kube-0-w1 iscsid[1632]: iscsid: disconnecting conn 0x56483ca0f050, fd 7 +Oct 30 16:23:02 pbm-kube-0-w1 iscsid[1632]: iscsid: connecting to 169.254.0.2:3260 +Oct 30 16:23:02 pbm-kube-0-w1 iscsid[1632]: iscsid: connect to 169.254.0.2:3260 failed (Connection refused) +Oct 30 16:23:02 pbm-kube-0-w1 iscsid[1632]: iscsid: deleting a scheduled/waiting thread! +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: Poll was woken by an alarm +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: re-opening session -1 (reopen_cnt 55046) +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: disconnecting conn 0x56483cb55e60, fd 9 +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: connecting to 169.254.0.2:3260 +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: connect to 169.254.0.2:3260 failed (Connection refused) +Oct 30 16:23:03 pbm-kube-0-w1 iscsid[1632]: iscsid: deleting a scheduled/waiting thread! +``` + +### NFS Environment + +Ensure that: + +* All worker nodes have the NFS packages installed for their Operating System: + + ```bash + $ yum install nfs-utils -y + ``` +* All worker nodes are running the daemon `rpc.statd` + +### Enabling Kubernetes Volume Snapshot Feature (Only for Kubernetes v1.17 - v1.19) + +The Kubernetes Volume Snapshot feature became GA in Kubernetes v1.20. In order to use +this feature in Kubernetes pre-v1.20, it MUST be enabled prior to deploying ZS CSI Driver. +To enable the feature on Kubernetes pre-v1.20, deploy API extensions, associated configurations, +and a snapshot controller by running the following command in deploy directory: + +```text +kubectl apply -R -f k8s-1.17/snapshot-controller +``` + +This command will report creation of resources and configuratios as follows: + +```text +customresourcedefinition.apiextensions.k8s.io/volumesnapshotclasses.snapshot.storage.k8s.io created +customresourcedefinition.apiextensions.k8s.io/volumesnapshotcontents.snapshot.storage.k8s.io created +customresourcedefinition.apiextensions.k8s.io/volumesnapshots.snapshot.storage.k8s.io created +serviceaccount/snapshot-controller created +clusterrole.rbac.authorization.k8s.io/snapshot-controller-runner created +clusterrolebinding.rbac.authorization.k8s.io/snapshot-controller-role created +role.rbac.authorization.k8s.io/snapshot-controller-leaderelection created +rolebinding.rbac.authorization.k8s.io/snapshot-controller-leaderelection created +statefulset.apps/snapshot-controller created +``` + +The details of them can be viewed using kubectl get command. Note that the command +above deploys a snapshot-controler in the default namespace by default. The command +`kubectl get all` should present something similar to this: + +```text +NAME READY STATUS RESTARTS AGE +pod/snapshot-controller-0 1/1 Running 0 5h22m +... +NAME READY AGE +statefulset.apps/snapshot-controller 1/1 5h22m +``` + +### CSI Volume Plugin Deployment from Helm + +A sample Helm chart is available in the deploy/helm directory, this method can be used for +simpler deployment than the section below. + +Create a local-values.yaml file that, at a minimum, sets the values for the +zfssaInformation section. Depending on your environment, the image block may +also need updates if the identified repositories cannot be reached. + +The secrets must be encoded. There are many ways to Base64 strings and files, +this technique would encode a user name of 'demo' for use in the values file on +a Mac with the base64 tool installed: + +```bash +echo -n 'demo' | base64 +``` +The following example shows how to get the server certificate of ZFSSA and encode it: +```bash +openssl s_client -connect :215 2>/dev/null + password: + ``` + For development only, other mechanisms can be used to create and share the secret with the container. + + *Warning* Do not store your credentials in source code control (such as this project). For production + environments use a secure secret store that encrypts at rest and can provide credentials through role + based access controls (refer to Kubernetes documentation). Do not use root user in production environments, + a purpose-built user will provide better audit controls and isolation for the driver. + +2. Create the Kubernetes secret containing the certificate chain for the appliance and make it available to the + driver in a mounted volume (/mnt/certs) and file name of zfssa.crt. While a certificate chain is a public + document, it is typically also provided by a volume mounted from a secret provider to protect the chain + of trust and bind it to the instance. + + To create a Kubernetes-secret from the certificate chain: + + ```text + kubectl create secret generic oracle.zfssa.csi.node.myappliance.certs -n myspace --from-file=./mycertfile + ``` + For development only, it is possible to run without the appliance chain of trust, see the options for + the driver. + +3. Update the deployment files. + + * __zfssa-csi-plugin.yaml__ + + In the `DaemonSet` section make the following modifications: + + * in the container _node-driver-registar_ subsection + * set `image` to the appropriate container image. + * in the container _zfssabs_ subsection + * set `image` for the container _zfssabs_ to the appropriate container image. + * in the `env` subsection + * under `ZFSSA_TARGET` set `valueFrom.secretKeyRef.name` to _oracle.zfssa.csi.node.myappliance_ + * under `ZFSSA_INSECURE` set `value` to _False_ (if you choose to **SECURE** the communication + with the appliance otherwise let it set to _True_) + * in the volume subsection, (skip if you set `ZFSSA_INSECURE` to _True_) + * under `cert`, **if you want communication with the appliance to be secure** + * set `secret.secretName` to _oracle.zfssa.csi.node.myappliance.certs_ + * set `secret.secretName.items.key` to _mycertfile_ + * set `secret.secretName.items.path` to _mycertfile_ + + * __zfssa-csi-provisioner.yaml__ + + In the `StatefulSet` section make the following modifications: + + * set `image` for the container _zfssa-csi-provisioner_ to the appropriate image. + * set `image` for the container _zfssa-csi-attacher_ to the appropriate container image. + +4. Deploy the plugin running the following commands: + + ```text + kubectl apply -n myspace -f ./zfssa-csi-rbac.yaml + kubectl apply -n myspace -f ./zfssa-csi-plugin.yaml + kubectl apply -n myspace -f ./zfssa-csi-provisioner.yaml + ``` + At this point the command `kubectl get all -n myspace` should return something similar to this: + ```text + NAME READY STATUS RESTARTS AGE + pod/zfssa-csi-nodeplugin-lpts9 2/2 Running 0 3m22s + pod/zfssa-csi-nodeplugin-vdb44 2/2 Running 0 3m22s + pod/zfssa-csi-provisioner-0 2/2 Running 0 72s + + NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE + daemonset.apps/zfssa-csi-nodeplugin 2 2 2 2 2 3m16s + + NAME READY AGE + statefulset.apps/zfssa-csi-provisioner 1/1 72s + ``` + +###Deployment Example Using an NFS Share + +Refer to the [NFS EXAMPLE README](./examples/nfs/README.md) file for details. + +###Deployment Example Using a Block Volume + +Refer to the [BLOCK EXAMPLE README](./examples/block/README.md) file for details. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..bbf4015 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,35 @@ +Copyright (c) 2021 Oracle. + +The Universal Permissive License (UPL), Version 1.0 + +Subject to the condition set forth below, permission is hereby granted to any +person obtaining a copy of this software, associated documentation and/or data +(collectively the "Software"), free of charge and under any and all copyright +rights in the Software, and any and all patent rights owned or freely +licensable by each licensor hereunder covering either (i) the unmodified +Software as contributed to or provided by such licensor, or (ii) the Larger +Works (as defined below), to deal in both + +(a) the Software, and +(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +one is included with the Software (each a "Larger Work" to which the Software +is contributed by such licensors), + +without restriction, including without limitation the rights to copy, create +derivative works of, display, perform, and distribute the Software and make, +use, sell, offer for sale, import, export, have made, and have sold the +Software and the Larger Work(s), and to sublicense the foregoing rights on +either these or other terms. + +This license is subject to the following condition: +The above copyright notice and either this complete permission notice or at +a minimum a reference to the UPL must be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8c8942e --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +# Copyright (c) 2021 Oracle and/or its affiliates. +# +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +CMDS=zfssa-csi-driver +all: build + +include release-tools/build.make diff --git a/README.md b/README.md index 1be7983..6177d87 100644 --- a/README.md +++ b/README.md @@ -1 +1,79 @@ -# zfssa-csi-driver +# About zfssa-cs-driver + +This plugin supports Oracle ZFS Storage Appliance +as a backend for block storage (iSCSI volumes) and file storage (NFS). + +| CSI Plugin Version | Supported CSI Versions | Supported Kubernetes Versions | Persistence | Supported Access Modes | Dynamic Provisioning | Raw Block Support | +| --- | --- | --- | --- | --- | --- | --- | +| v0.5.x | v1.0+ | v1.17.X+ | Persistent | Read/Write Once (for Block) | Yes | Yes | + +## Requirements + +* Kubernetes v1.17 or above. +* A Container runtime implementing the Kubernetes Container Runtime Interface. This plugin was tested with CRI-O v1.17. +* An Oracle ZFS Storage Appliance running Appliance Kit Version 8.8 or above. This plugin may work with previous +versions but it is not tested with them. It is possible to use this +driver with the [Oracle ZFS Storage Simulator](https://www.oracle.com/downloads/server-storage/sun-unified-storage-sun-simulator-downloads.html) +* Access to both a management path and a data path for the target Oracle +ZFS Storage Appiance (or simulator). The management and data path +can be the same address. +* A suitable container image build environment. The Makefile currently uses docker +but with minor updates to release-tools/build.make, podman should also be usable. + +## Unsupported Functionality +ZFS Storage Appliance CSI driver does not support the following functionality: +* Volume Cloning + +## Building + +Use and enhance the Makefile in the root directory and release-tools/build.make. + +``` +make +``` + +## Installation + +See [INSTALLATION](./INSTALLATION.md) for details. + +## Testing + +For information about testing the driver, see [TEST](./TEST.md). + +## Examples + +Example usage of this driver can be found in the ./examples +directory. + +The examples below use the image _container-registry.oracle.com/os/oraclelinux:7-slim_ +when they create a container where a persistent volume(s) is attached and mounted. + +This set uses dynamic volume creation. +* [NFS](./examples/nfs/README.md) - illustrates NFS volume usage +from a simple container. +* [Block](./examples/block/README.md) - illustrates block volume +usage from a simple container. +* [NFS multi deployment](./examples/nfs-multi) - illustrates the use +of Helm to build several volumes and optionally build a pod to consume them. + +This next set uses existing shares on the target appliance: +* [Existing NFS](./examples/nfs-pv/README.md) - illustrates NFS volume usage +from a simple container of an existing NFS filesystem share. +* [Existing Block](./examples/block-pv/README.md) - illustrates block volume +usage from a simple container of an existing iSCSI LUN. + +This set exercises dynamic volume creation followed by expanding the volume capacity. +* [NFS Volume Expansion](./examples/nfs-exp/README.md) - illustrates an expansion of an NFS volume. + +This set exercises dynamic volume creation (restoring from a volume snapshot) followed by creating a snapshot of the volume. +* [NFS Volume Snapshot](./examples/nfs-vsc/README.md) - illustrates a snapshot creation of an NFS volume. +* [Block Volume Snapshot](./examples/block-vsc/README.md) - illustrates a snapshot creation of a block volume. + +## Help + +Refer to the documentation links and examples for more information on +this driver. + +## Contributing + +See [CONTRIBUTING](./CONTRIBUTING.md) for details. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2cea12b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Reporting Security Vulnerabilities + +Oracle values the independent security research community and believes that responsible disclosure of security vulnerabilities helps us ensure the security and privacy of all our users. + +Please do NOT raise a GitHub Issue to report a security vulnerability. If you believe you have found a security vulnerability, please submit a report to secalert_us@oracle.com preferably with a proof of concept. We provide additional information on [how to report security vulnerabilities to Oracle](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html) which includes public encryption keys for secure email. + +We ask that you do not use other channels or contact project contributors directly. + +Non-vulnerability related security issues such as new great new ideas for security features are welcome on GitHub Issues. + +## Security Updates, Alerts and Bulletins + +Security updates will be released on a regular cadence. Many of our projects will typically release security fixes in conjunction with the [Oracle Critical Patch Update](https://www.oracle.com/security-alerts/) program. Security updates are released on the Tuesday closest to the 17th day of January, April, July and October. A pre-release announcement will be published on the Thursday preceding each release. Additional information, including past advisories, is available on our [Security Alerts](https://www.oracle.com/security-alerts/) page. + +## Security-Related Information + +We will provide security related information such as a threat model, considerations for secure use, or any known security issues in our documentation. Please note that labs and sample code are intended to demonstrate a concept and may not be sufficiently hardened for production use. diff --git a/TEST.md b/TEST.md new file mode 100644 index 0000000..f6df9a7 --- /dev/null +++ b/TEST.md @@ -0,0 +1,139 @@ +# Oracle ZFS Storage Appliance CSI Plugin Testing + +There are two distinct paths in the plugin +* Filesystem (Mount) +* Block (iSCSI) + +This write-up discusses various techniques for testing the plugin. + +Note that some of the paths require the plug-in to be deployed (unit tests) while others are more for quick iterative development without +a cluster being available. + +Refer to the README.md file for information about deploying the plug-in. + +## Test Driver Locally + +Create a local build +``` +make build +``` + +You will need the following environment variables configured (these emulate the secrets setup). Note that +the driver *must* be run as root (or with root-equivalence for iscsiadmin utility). The csc tool can be run +from any process space as it only interacts with the socket that the driver is waiting on. + +``` +ZFSSA_TARGET=(target zfssa head) +ZFSSA_USER=(user for target) +ZFSSA_PASSWORD=(password for target) +ZFSSA_POOL=(pool to create resources in) +ZFSSA_PROJECT=(project to create resources in) +HOST_IP=(IP address of host where running) +POD_IP=(IP of pod) +NODE_NAME=(name of node) +CSI_ENDPOINT=tcp://127.0.0.1:10000 +``` + +Now run the driver as root: + +``` +sudo su - +export CSI_ENDPOINT=tcp://127.0.0.1:10000;;./bin/zfssa-csi-driver --endpoint tcp://127.0.0.1:10000 --nodeid MyCSINode +Building Provider +ERROR: logging before flag.Parse: I1108 15:34:55.472389 6622 service.go:63] Driver: zfssa-csi-driver version: 0.0.0 +Using stored configuration { } +ERROR: logging before flag.Parse: I1108 15:34:55.472558 6622 controller.go:42] NewControllerServer Implementation +ERROR: logging before flag.Parse: I1108 15:34:55.472570 6622 identity.go:15] NewIdentityServer Implementation +ERROR: logging before flag.Parse: I1108 15:34:55.472581 6622 node.go:24] NewNodeServer Implementation +Running gRPC +INFO[0000] identity service registered +INFO[0000] controller service registered +INFO[0000] node service registered +INFO[0000] serving endpoint="tcp://127.0.0.1:10000" + +``` + +Test the block driver. Note the interactions of using the full volume id that is returned from each command. +``` +./csc controller create-volume --cap MULTI_NODE_MULTI_WRITER,block --req-bytes 1073741824 --params pool=dedup1,project=pmonday,initiatorGroup=paulGroup,targetGroup=paulTargetGroup,blockSize=8192 --endpoint tcp://127.0.0.1:10000 pmonday5 +./csc controller delete-volume --endpoint tcp://127.0.0.1:10000 /iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday3 +``` + +Test the driver to Publish a LUN. Note that the LUN number is the number of the "assignednumber" on the +LUN on the appliance. Also note that the flow is create -> controller publish -> node publish -> node unpublish -> controller unpublish +```bash +First publish to the controller +./csc controller publish --cap MULTI_NODE_MULTI_WRITER,block --vol-context targetPortal=10.80.44.165:3260,discoveryCHAPAuth=false,sessionCHAPAuth=false,portals=[],iscsiInterface=default --node-id worknode /iscsi/aie-7330a-h1/pmonday5/dedup1/local/pmonday/pmonday5 +"/iscsi/aie-7330a-h1/pmonday5/dedup1/local/pmonday/pmonday5" "devicePath"="/dev/disk/by-path/ip-10.80.44.165:3260-iscsi-iqn.1986-03.com.sun:02:ab7b55fa-53ee-e5ab-98e1-fad3cc29ae57-lun-13" + +Publish to the Node +./csc node publish -l debug --endpoint tcp://127.0.0.1:10000 --target-path /mnt/iscsi --pub-context "devicePath"="/dev/disk/by-path/ip-10.80.44.165:3260-iscsi-iqn.1986-03.com.sun:02:ab7b55fa-53ee-e5ab-98e1-fad3cc29ae57-lun-13" --cap MULTI_NODE_MULTI_WRITER,block --vol-context targetPortal=10.80.44.165:3260,discoveryCHAPAuth=false,sessionCHAPAuth=false,portals=[],iscsiInterface=default /iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 +DEBU[0000] assigned the root context +DEBU[0000] mounting volume request="{/iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 map[devicePath:/dev/disk/by-path/ip-10.80.44.165:3260-iscsi-iqn.1986-03.com.sun:02:ab7b55fa-53ee-e5ab-98e1-fad3cc29ae57-lun-13] /mnt/iscsi block:<> access_mode: false map[] map[discoveryCHAPAuth:false iscsiInterface:default portals:[] sessionCHAPAuth:false targetPortal:10.80.44.165:3260] {} [] 0}" +DEBU[0000] parsed endpoint info addr="127.0.0.1:10000" proto=tcp timeout=1m0s +/iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 + +Now unpublish from the node +./csc node unpublish -l debug --endpoint tcp://127.0.0.1:10000 --target-path /mnt/iscsi /iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 +DEBU[0000] assigned the root context +DEBU[0000] mounting volume request="{/iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 /mnt/iscsi {} [] 0}" +DEBU[0000] parsed endpoint info addr="127.0.0.1:10000" proto=tcp timeout=1m0s +/iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 + +Now unpublish from the controller (this is not working yet) +./csc controller unpublish -l debug --endpoint tcp://127.0.0.1:10000 /iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 +DEBU[0000] assigned the root context +DEBU[0000] unpublishing volume request="{/iscsi/aie-7330a-h1/pmonday3/dedup1/local/pmonday/pmonday5 map[] {} [] 0}" +DEBU[0000] parsed endpoint info addr="127.0.0.1:10000" proto=tcp timeout=1m0s +Need to just detach the device here + +``` +If everything looks OK, push the driver to a container registry +``` +make push +``` + +Now deploy the driver in a Kubernetes Cluster +```bash +Working on instructions +``` + +Test the driver to Create a file system +``` +./csc controller create --cap MULTI_NODE_MULTI_WRITER,mount,nfs,uid=500,gid=500 --req-bytes 107374182400 --params node=zs32-01,pool=p0,project=default coucou +./csc controller delete /nfs/10.80.222.176/coucou/p0/local/default/coucou +``` + +## Unit Test the Driver + +To run the unit tests, you must have compiled csi-sanity from the +[csi-test project](https://github.com/kubernetes-csi/csi-test). Once +compiled, scp the file to the node that is functioning as a controller +in a Kubernetes Cluster with the ZFSSA CSI Driver deployed and running. + +There is documentation on the test process available at the Kubernetes +[testing of CSI drivers page](https://kubernetes.io/blog/2020/01/08/testing-of-csi-drivers/). + +At least do a quick sanity test (create a PVC and remove it) prior to running. +This will shake out simple problems like authentication. + +Create a test parameters file that makes sense in your environment, like this: +```yaml +volumeType: thin +targetGroup: csi-data-path-target +blockSize: "8192" +pool: h1-pool1 +project: pmonday +targetPortal: "10.80.44.65:3260" +nfsServer: "10.80.44.65" +``` +Then run the tests +``` +./csi-sanity --csi.endpoint=/var/lib/kubelet/plugins/com.oracle.zfssabs/csi.sock -csi.testvolumeparameters=./test-volume-parameters.yaml +``` + +There should only be one known failure due to volume name length limitations +on the Oracle ZFS Storage Appliance, it looks like this: +``` +[Fail] Controller Service [Controller Server] CreateVolume [It] should not fail when creating volume with maximum-length name +``` diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt new file mode 100644 index 0000000..0deb23f --- /dev/null +++ b/THIRD_PARTY_LICENSES.txt @@ -0,0 +1,2579 @@ + +github.com/container-storage-interface/spec +version: v1.2.0 +COPYRIGHT and LICENSE: Apache License 2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +github.com/golang/groupcache +version: 5b532d6fd5ef +COPYRIGHT and LICENSE: Apache License 2.0 +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +github.com/golang/protobuf +version: v1.4.0 +COPYRIGHT and LICENSE: BSD 3-Clause "New" or "Revised" License +Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +--------------------------------------------------------------------------------- +github.com/kubernetes-csi/csi-lib-iscsi +version: c545557492f4 +COPYRIGHT and LICENSE: Apache License 2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +github.com/kubernetes-csi/csi-lib-utils +version: v0.6.1 +COPYRIGHT and LICENSE: Apache License 2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +github.com/onsi/gomega +version: v1.9.0 +COPYRIGHT and LICENSE: MIT License +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------------------------------- +github.com/prometheus/client_golang +version: v1.2.1 +COPYRIGHT and LICENSE: Apache License 2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +golang.org/x/net +version: 0deb6923b6d9 +COPYRIGHT and LICENSE: BSD-3-Clause +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--------------------------------------------------------------------------------- +golang.org/x/sys +version: d5e6a3e2c0ae +COPYRIGHT and LICENSE: BSD-3-Clause +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--------------------------------------------------------------------------------- +google.golang.org/grpc +version: v1.23.1 +COPYRIGHT and LICENSE: Apache License 2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +gopkg.in/yaml.v2 +version: v2.2.8 +COPYRIGHT and LICENSE: Apache-2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +k8s.io/apimachinery +version: v0.17.11 +COPYRIGHT and LICENSE: Apache License 2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +k8s.io/client-go +version: v0.18.2 +COPYRIGHT and LICENSE: Apache License 2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +k8s.io/klog +version: v1.0.0 +COPYRIGHT and LICENSE: Apache License 2.0 +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +k8s.io/kubernetes +version: v1.17.5 +COPYRIGHT and LICENSE: Apache License 2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------------------------------------------------------------------------- +k8s.io/utils +version: e782cd3c129f +COPYRIGHT and LICENSE: Apache License 2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/cmd/zfssa-csi-driver/main.go b/cmd/zfssa-csi-driver/main.go new file mode 100644 index 0000000..405671f --- /dev/null +++ b/cmd/zfssa-csi-driver/main.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +*/ + +package main + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/service" + "flag" + "fmt" + "os" +) + +var ( + driverName = flag.String("drivername", "zfssa-csi-driver", "name of the driver") + // Provided by the build process + version = "0.0.0" +) + +func main() { + + zd, err := service.NewZFSSADriver(*driverName, version) + if err != nil { + fmt.Print(err) + } else { + zd.Run() + } + os.Exit(1) +} diff --git a/deploy/helm/k8s-1.17/Chart.yaml b/deploy/helm/k8s-1.17/Chart.yaml new file mode 100644 index 0000000..ee1c303 --- /dev/null +++ b/deploy/helm/k8s-1.17/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi +version: 0.0.2 +description: Deploys Oracle ZFS Storage Appliance CSI Plugin. diff --git a/deploy/helm/k8s-1.17/templates/00-zfssa-csi-secret.yaml b/deploy/helm/k8s-1.17/templates/00-zfssa-csi-secret.yaml new file mode 100644 index 0000000..e27c647 --- /dev/null +++ b/deploy/helm/k8s-1.17/templates/00-zfssa-csi-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +stringData: + zfssa.yaml: | + username: {{ .Values.zfssaInformation.username }} + password: {{ .Values.zfssaInformation.password }} +kind: Secret +metadata: + name: oracle.zfssa.csi.node + namespace: {{ .Values.deployment.namespace }} diff --git a/deploy/helm/k8s-1.17/templates/01-zfssa-csi-certs.yaml b/deploy/helm/k8s-1.17/templates/01-zfssa-csi-certs.yaml new file mode 100644 index 0000000..977d273 --- /dev/null +++ b/deploy/helm/k8s-1.17/templates/01-zfssa-csi-certs.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + zfssa.crt: {{ .Values.zfssaInformation.cert }} +kind: Secret +metadata: + name: oracle.zfssa.csi.node.certs +type: Opaque + diff --git a/deploy/helm/k8s-1.17/templates/02-zfssa-csi-rbac.yaml b/deploy/helm/k8s-1.17/templates/02-zfssa-csi-rbac.yaml new file mode 100644 index 0000000..3c4a841 --- /dev/null +++ b/deploy/helm/k8s-1.17/templates/02-zfssa-csi-rbac.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zfssa-csi +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zfssa-csi-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch", "delete", "get"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update" ] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: zfssa-csi-role-binding +subjects: + - kind: ServiceAccount + name: zfssa-csi + namespace: {{ .Values.deployment.namespace }} +roleRef: + kind: ClusterRole + name: zfssa-csi-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/helm/k8s-1.17/templates/03-zfssa-csi-driver.yaml b/deploy/helm/k8s-1.17/templates/03-zfssa-csi-driver.yaml new file mode 100644 index 0000000..67ae164 --- /dev/null +++ b/deploy/helm/k8s-1.17/templates/03-zfssa-csi-driver.yaml @@ -0,0 +1,139 @@ +# Service defined here, plus serviceName below in StatefulSet, +# are needed only because of condition explained in +# https://github.com/kubernetes/kubernetes/issues/69608 +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: zfssa-csi-driver + namespace: {{ .Values.deployment.namespace }} +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: zfssa-csi-nodeplugin + namespace: {{ .Values.deployment.namespace }} +spec: + selector: + matchLabels: + app: zfssa-csi-nodeplugin + template: + metadata: + labels: + app: zfssa-csi-nodeplugin + spec: + serviceAccount: zfssa-csi + hostNetwork: true + containers: + - name: node-driver-registrar + image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiNodeDriverRegistrar.name }}:{{ .Values.images.csiNodeDriverRegistrar.tag }} + args: + - --v=5 + - --csi-address=/plugin/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/com.oracle.zfssabs/csi.sock + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.paths.pluginDir.mountPath }} + - name: registration-dir + mountPath: /registration + + - name: zfssabs + image: {{ .Values.image.zfssaBase }}{{ .Values.images.zfssaCsiDriver.name }}:{{ .Values.images.zfssaCsiDriver.tag }} + args: + - "--drivername=zfssa-csi-driver.oracle.com" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + - name: LOG_LEVEL + value: "5" + - name: ZFSSA_TARGET + value: {{ .Values.zfssaInformation.target }} + - name: ZFSSA_INSECURE + value: "False" + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.paths.pluginDir.mountPath }} + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + - name: plugins-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + - name: dev-dir + mountPath: /dev + - name: zfssa-credentials + mountPath: "/mnt/zfssa" + readOnly: true + - name: certs + mountPath: "/mnt/certs" + readOnly: true + volumes: + - name: socket-dir + hostPath: + path: {{ .Values.paths.pluginDir.hostPath }} + type: DirectoryOrCreate + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: plugins-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: dev-dir + hostPath: + path: /dev + type: Directory + - name: zfssa-credentials + secret: + secretName: oracle.zfssa.csi.node + items: + - key: zfssa.yaml + path: zfssa.yaml + - name: certs + secret: + secretName: oracle.zfssa.csi.node.certs + items: + - key: zfssa.crt + path: zfssa.crt diff --git a/deploy/helm/k8s-1.17/templates/04-zfssa-csi-provisioner.yaml b/deploy/helm/k8s-1.17/templates/04-zfssa-csi-provisioner.yaml new file mode 100644 index 0000000..f544543 --- /dev/null +++ b/deploy/helm/k8s-1.17/templates/04-zfssa-csi-provisioner.yaml @@ -0,0 +1,90 @@ +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: zfssa-csi-provisioner + namespace: {{ .Values.deployment.namespace }} +spec: + serviceName: "zfssa-csi-provisioner" + replicas: 1 + selector: + matchLabels: + app: zfssa-csi-provisioner + template: + metadata: + labels: + app: zfssa-csi-provisioner + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - zfssa-csi-nodeplugin + topologyKey: kubernetes.io/hostname + serviceAccountName: zfssa-csi + containers: + - name: zfssa-csi-snapshotter + image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiSnapshotter.name }}:{{ .Values.images.csiSnapshotter.tag }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=false" + env: + - name: ADDRESS + value: /plugin/csi.sock + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /plugin + - name: zfssa-csi-resizer + image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiResizer.name }}:{{ .Values.images.csiResizer.tag }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /plugin/csi.sock + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /plugin + - name: zfssa-csi-provisioner + image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiProvisioner.name }}:{{ .Values.images.csiProvisioner.tag }} + args: + - -v=5 + - --csi-address=/plugin/csi.sock + - --timeout=30s + - --feature-gates=Topology=true + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /plugin + - name: zfssa-csi-attacher + image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiAttacher.name }}:{{ .Values.images.csiAttacher.tag }} + args: + - --v=5 + - --csi-address=/plugin/csi.sock + # securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + # privileged: true + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.paths.pluginDir.mountPath }} + volumes: + - name: socket-dir + hostPath: + path: {{ .Values.paths.pluginDir.hostPath }} + type: DirectoryOrCreate \ No newline at end of file diff --git a/deploy/helm/k8s-1.17/values.yaml b/deploy/helm/k8s-1.17/values.yaml new file mode 100644 index 0000000..364f9df --- /dev/null +++ b/deploy/helm/k8s-1.17/values.yaml @@ -0,0 +1,42 @@ +# Global docker image setting +image: + sidecarBase: k8s.gcr.io/sig-storage/ + zfssaBase: iad.ocir.io/zs/store/csi/ + pullPolicy: Always + +# Define all the images that will be used during helm chart deployment +images: + csiNodeDriverRegistrar: + name: csi-node-driver-registrar + tag: "v2.0.0" + zfssaCsiDriver: + name: zfssa-csi-driver + tag: "v1.0.0" + csiProvisioner: + name: csi-provisioner + tag: "v2.0.5" + csiAttacher: + name: csi-attacher + tag: "v3.0.2" + csiResizer: + name: csi-resizer + tag: "v1.1.0" + csiSnapshotter: + name: csi-snapshotter + tag: "v3.0.3" + +paths: + pluginDir: + hostPath: "/var/lib/kubelet/plugins/com.oracle.zfssabs" + mountPath: "/plugin" + +deployment: + namespace: default + +# ZFSSA-specific information +# It is desirable to provision a normal login user with required authorizations. +zfssaInformation: + username: text-string + password: text-string + target: text-string + cert: cert-base64-encoded diff --git a/deploy/helm/local-values/README.md b/deploy/helm/local-values/README.md new file mode 100644 index 0000000..99a1a62 --- /dev/null +++ b/deploy/helm/local-values/README.md @@ -0,0 +1,5 @@ +# Introduction + +This directory can contain local values files to be used with the helm charts. + +Files in this directory should not be checked in to a source code control. diff --git a/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 0000000..4aa980c --- /dev/null +++ b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,85 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .driver + name: Driver + type: string + - JSONPath: .deletionPolicy + description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass + should be deleted when its bound VolumeSnapshot is deleted. + name: DeletionPolicy + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + preserveUnknownFields: false + scope: Cluster + subresources: {} + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 0000000..34c51ad --- /dev/null +++ b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,233 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot in bytes + name: RestoreSize + type: integer + - JSONPath: .spec.deletionPolicy + description: Determines whether this VolumeSnapshotContent and its physical snapshot + on the underlying storage system should be deleted when its bound VolumeSnapshot + is deleted. + name: DeletionPolicy + type: string + - JSONPath: .spec.driver + description: Name of the CSI driver used to create the physical snapshot on the + underlying storage system. + name: Driver + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: Name of the VolumeSnapshotClass to which this snapshot belongs. + name: VolumeSnapshotClass + type: string + - JSONPath: .spec.volumeSnapshotRef.name + description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + name: VolumeSnapshot + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + preserveUnknownFields: false + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshots.yaml b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 0000000..483706f --- /dev/null +++ b/deploy/k8s-1.17/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,188 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .spec.source.persistentVolumeClaimName + description: Name of the source PVC from where a dynamically taken snapshot will + be created. + name: SourcePVC + type: string + - JSONPath: .spec.source.volumeSnapshotContentName + description: Name of the VolumeSnapshotContent which represents a pre-provisioned + snapshot. + name: SourceSnapshotContent + type: string + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot. + name: RestoreSize + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + name: SnapshotClass + type: string + - JSONPath: .status.boundVolumeSnapshotContentName + description: The name of the VolumeSnapshotContent to which this VolumeSnapshot + is bound. + name: SnapshotContent + type: string + - JSONPath: .status.creationTime + description: Timestamp when the point-in-time snapshot is taken by the underlying + storage system. + name: CreationTime + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + anyOf: + - type: integer + - type: string + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/k8s-1.17/snapshot-controller/rbac-snapshot-controller.yaml b/deploy/k8s-1.17/snapshot-controller/rbac-snapshot-controller.yaml new file mode 100644 index 0000000..fee85af --- /dev/null +++ b/deploy/k8s-1.17/snapshot-controller/rbac-snapshot-controller.yaml @@ -0,0 +1,80 @@ +# RBAC file for the snapshot controller. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default # TODO: replace with the namespace you want for your controller + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: default # TODO: replace with the namespace you want for your controller +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: default # TODO: replace with the namespace you want for your controller +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/deploy/k8s-1.17/snapshot-controller/setup-snapshot-controller.yaml b/deploy/k8s-1.17/snapshot-controller/setup-snapshot-controller.yaml new file mode 100644 index 0000000..f958d2c --- /dev/null +++ b/deploy/k8s-1.17/snapshot-controller/setup-snapshot-controller.yaml @@ -0,0 +1,26 @@ +# This YAML file shows how to deploy the snapshot controller + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller +spec: + serviceName: "snapshot-controller" + replicas: 1 + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.1.1 + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: Always diff --git a/examples/block-pv/Chart.yaml b/examples/block-pv/Chart.yaml new file mode 100644 index 0000000..b3e153c --- /dev/null +++ b/examples/block-pv/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-existing-block-example +version: 0.0.1 +description: Deploys an end to end iSCSI volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/block-pv/README.md b/examples/block-pv/README.md new file mode 100644 index 0000000..9cc530b --- /dev/null +++ b/examples/block-pv/README.md @@ -0,0 +1,110 @@ +# Introduction + +This is an end-to-end example of using an existing iSCSI block devices on a target +Oracle ZFS Storage Appliance. + +Prior to running this example, the iSCSI environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +This flow to use an existing volume is: +* create a persistent volume (PV) object +* allocate it to a persistent volume claim (PVC) +* use the PVC from a pod + +The following must be set up: +* the volume handle must be a fully formed volume id +* there must be volume attributes defined as part of the persistent volume +* the initiator group for the block volume *must* be set to ```com.sun.ms.vss.hg.maskAll``` + +In this example, the volume handle is constructed via values in the helm +chart. The only new attribute necessary is the name of the volume on the +target appliance. The remaining is assembled from the information that is still +in the local-values.yaml file (appliance name, pool, project, etc...). + +The resulting VolumeHandle appears similar to the following, with the values +in ```<>``` filled in from the helm variables: + +``` + volumeHandle: /iscsi////local// +``` +From the above, note that the volumeHandle is in the form of an ID with the components: +* 'iscsi' - denoting a block volume +* 'appliance name' - this is the management path of the ZFSSA target appliance +* 'volume name' - the name of the share on the appliance +* 'pool name' - the pool on the target appliance +* 'local' - denotes that the pool is owned by the head +* 'project' - the project that the share is in + +In the volume attributes, the targetGroup and targetPortal must be defined. This should be similar +to that in the storage class. + +Once created, a persistent volume claim can be made for this share and used in a pod. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * targetGroup: the target iSCSI group to use on the appliance + * nfsServer: the NFS data path IP address +* applianceName: name of the appliance +* pvExistingName: the name of the iSCSI LUN share on the target appliance +* volSize: the size of the iSCSI LUN share specified by pvExistingName + +On the target appliance, locate the share via the CLI: + +``` +appliance> shares +appliance> select +appliance> +appliance> +appliance> set initatorgroup=com.sun.ms.vss.hg.maskAll +appliance> commit +``` + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f ../local-values/local-values.yaml zfssa-block-existing ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +``` +kubectl get sc +kubectl get pvc +kubectl get pod +``` + +## Writing data + +Once the pod is deployed, for demo, start the following analytics in a worksheet on +the Oracle ZFS Storage Appliance that is hosting the target LUNs: + +* Protocol -> iSCSI bytes broken down by initiator +* Protocol -> iSCSI bytes broken down by target +* Protocol -> iSCSI bytes broken down by LUN + +Exec into the pod and write some data to the block volume: +```yaml +kubectl exec -it zfssa-block-existing-pod -- /bin/sh +/ # cd /dev +/dev # ls +block fd mqueue ptmx random stderr stdout tty zero +core full null pts shm stdin termination-log urandom +/dev # dd if=/dev/zero of=/dev/block count=1024 bs=1024 +1024+0 records in +1024+0 records out +/dev # +``` + +The analytics on the appliance should have seen the spikes as data was written. diff --git a/examples/block-pv/templates/00-storage-class.yaml b/examples/block-pv/templates/00-storage-class.yaml new file mode 100644 index 0000000..70d6865 --- /dev/null +++ b/examples/block-pv/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scBlockName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: "on" + restrictChown: "false" diff --git a/examples/block-pv/templates/01-pv.yaml b/examples/block-pv/templates/01-pv.yaml new file mode 100644 index 0000000..c467400 --- /dev/null +++ b/examples/block-pv/templates/01-pv.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .Values.pvExistingName }} + annotations: + pv.kubernetes.io/provisioned-by: zfssa-csi-driver +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + persistentVolumeReclaimPolicy: Retain + capacity: + storage: {{ .Values.volSize }} + csi: + driver: zfssa-csi-driver + volumeHandle: /iscsi/{{ .Values.applianceName }}/{{ .Values.pvExistingName }}/{{ .Values.appliance.pool }}/local/{{ .Values.appliance.project }}/{{ .Values.pvExistingName }} + readOnly: false + volumeAttributes: + targetGroup: {{ .Values.appliance.targetGroup }} + targetPortal: {{ .Values.appliance.targetPortal }} + claimRef: + namespace: default + name: {{ .Values.pvcExistingName }} + diff --git a/examples/block-pv/templates/02-pvc.yaml b/examples/block-pv/templates/02-pvc.yaml new file mode 100644 index 0000000..135f9d2 --- /dev/null +++ b/examples/block-pv/templates/02-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcExistingName }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scBlockName }} diff --git a/examples/block-pv/templates/03-pod.yaml b/examples/block-pv/templates/03-pod.yaml new file mode 100644 index 0000000..2fd090b --- /dev/null +++ b/examples/block-pv/templates/03-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podBlockName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeDevices: + - name: vol + devicePath: /dev/block + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcExistingName }} diff --git a/examples/block-pv/values.yaml b/examples/block-pv/values.yaml new file mode 100644 index 0000000..06a3a4a --- /dev/null +++ b/examples/block-pv/values.yaml @@ -0,0 +1,20 @@ +# Various names used through example +scBlockName: zfssa-block-existing-sc +pvExistingName: OVERRIDE +pvcExistingName: zfssa-block-existing-pvc +podBlockName: zfssa-block-existing-pod +applianceName: OVERRIDE + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + +# Settings for volume +volSize: OVERRIDE diff --git a/examples/block-snapshot/block-pod-restored-volume.yaml b/examples/block-snapshot/block-pod-restored-volume.yaml new file mode 100644 index 0000000..e4e815a --- /dev/null +++ b/examples/block-snapshot/block-pod-restored-volume.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: zfssa-block-vs-restore-pod + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeDevices: + - name: vol + devicePath: /dev/block + volumes: + - name: vol + persistentVolumeClaim: + claimName: zfssa-block-vs-restore-pvc + readOnly: false diff --git a/examples/block-snapshot/block-pvc-from-snapshot.yaml b/examples/block-snapshot/block-pvc-from-snapshot.yaml new file mode 100644 index 0000000..cea2be9 --- /dev/null +++ b/examples/block-snapshot/block-pvc-from-snapshot.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: zfssa-block-vs-restore-pvc +spec: + storageClassName: zfssa-block-vs-example-sc + dataSource: + name: zfssa-block-vs-snapshot + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: 68796 diff --git a/examples/block-snapshot/block-snapshot.yaml b/examples/block-snapshot/block-snapshot.yaml new file mode 100644 index 0000000..fa04c54 --- /dev/null +++ b/examples/block-snapshot/block-snapshot.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshot +metadata: + name: zfssa-block-vs-snapshot +spec: + volumeSnapshotClassName: zfssa-block-vs-example-vsc + source: + persistentVolumeClaimName: zfssa-block-vs-example-pvc diff --git a/examples/block-vsc/Chart.yaml b/examples/block-vsc/Chart.yaml new file mode 100644 index 0000000..2becf53 --- /dev/null +++ b/examples/block-vsc/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi-block-example +version: 0.0.1 +description: Deploys an end to end iSCSI volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/block-vsc/README.md b/examples/block-vsc/README.md new file mode 100644 index 0000000..b0859e8 --- /dev/null +++ b/examples/block-vsc/README.md @@ -0,0 +1,193 @@ +# Introduction + +This is an end-to-end example of taking a snapshot of a block volume (iSCSI Lun) +on a target Oracle ZFS Storage Appliance and making use of it +on another pod by creating (restoring) a volume from the snapshot. + +Prior to running this example, the iSCSI environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can contain others. The minimum set of values to +customize are: + +* appliance: + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * targetGroup: the target iSCSI group to use on the appliance +* volSize: the size of the iSCSI LUN share to create + +## Enabling Volume Snapshot Feature (Only for Kubernetes v1.17 - v1.19) + +The Kubernetes Volume Snapshot feature became GA in Kubernetes v1.20. In order to use +this feature in Kubernetes pre-v1.20, it MUST be enabled prior to deploying ZS CSI Driver. +To enable the feature on Kubernetes pre-v1.20, follow the instructions on +[INSTALLATION](../../INSTALLATION.md). + +## Deployment + +This step includes deploying a pod with a block volume attached using a regular +storage class and a persistent volume claim. It also deploys a volume snapshot class +required to take snapshots of the persistent volume. + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +```text +helm ../install -f local-values/local-values.yaml zfssa-block-vsc ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +1. Display the storage class (SC) + The command `kubectl get sc` should now return something similar to this: + + ```text + NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE + zfssa-block-vs-example-sc zfssa-csi-driver Delete Immediate false 86s + ``` +2. Display the volume claim + The command `kubectl get pvc` should now return something similar to this: + ```text + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + zfssa-block-vs-example-pvc Bound pvc-477804b4-e592-4039-a77c-a1c99a1e537b 10Gi RWO zfssa-block-vs-example-sc 62s + ``` +3. Display the volume snapshot class + The command `kubectl get volumesnapshotclass` should now return something similar to this: + ```text + NAME DRIVER DELETIONPOLICY AGE + zfssa-block-vs-example-vsc zfssa-csi-driver Delete 100s + ``` +4. Display the pod mounting the volume + + The command `kubectl get pod` should now return something similar to this: + ```text + NAME READY STATUS RESTARTS AGE + snapshot-controller-0 1/1 Running 0 14d + zfssa-block-vs-example-pod 1/1 Running 0 2m11s + zfssa-csi-nodeplugin-7kj5m 2/2 Running 0 3m11s + zfssa-csi-nodeplugin-rgfzf 2/2 Running 0 3m11s + zfssa-csi-provisioner-0 4/4 Running 0 3m11s + ``` + +## Writing data + +Once the pod is deployed, verify the block volume is mounted and can be written. + +```text +kubectl exec -it zfssa-block-vs-example-pod -- /bin/sh + +/ # cd /dev +/dev # +/dev # date > block +/dev # dd if=block bs=64 count=1 +Wed Jan 27 22:06:36 UTC 2021 +1+0 records in +1+0 records out +/dev # +``` +Alternatively, `cat /dev/block` followed by `CTRL-C` can be used to view the timestamp written on th /dev/block device file. + +## Creating snapshot + +Use configuration files in examples/block-snapshot directory with proper modifications +for the rest of the example steps. + +Create a snapshot of the volume by running the command below: + +```text +kubectl apply -f ../block-snapshot/block-snapshot.yaml +``` + +Verify the volume snapshot is created and available by running the following command: + +```text +kubectl get volumesnapshot +``` + +Wait until the READYTOUSE of the snapshot becomes true before moving on to the next steps. +It is important to use the RESTORESIZE value of the volume snapshot just created when specifying +the storage capacity of a persistent volume claim to provision a persistent volume using this +snapshot. For example, the storage capacity in ../block-snapshot/block-pvc-from-snapshot.yaml + +Optionally, verify the volume snapshot exists on ZFS Storage Appliance. The snapshot name +on ZFS Storage Appliance should have the volume snapshot UID as the suffix. + +## Creating persistent volume claim + +Create a persistent volume claim to provision a volume from the snapshot by running +the command below. Be aware that the persistent volume provisioned by this persistent volume claim +is not expandable. Create a new storage class with allowVolumeExpansion: true and use it when +specifying the persistent volume claim. + +```text +kubectl apply -f ../block-snapshot/block-pvc-from-snapshot.yaml +``` + +Verify the persistent volume claim is created and a volume is provisioned by running the following command: + +```text +kubectl get pv,pvc +``` + +The command `kubectl get pv,pvc` should return something similar to this: +```text +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +persistentvolume/pvc-477804b4-e592-4039-a77c-a1c99a1e537b 10Gi RWO Delete Bound default/zfssa-block-vs-example-pvc zfssa-block-vs-example-sc 13m +persistentvolume/pvc-91f949f6-5d77-4183-bab5-adfdb1452a90 10Gi RWO Delete Bound default/zfssa-block-vs-restore-pvc zfssa-block-vs-example-sc 11s + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/zfssa-block-vs-example-pvc Bound pvc-477804b4-e592-4039-a77c-a1c99a1e537b 10Gi RWO zfssa-block-vs-example-sc 13m +persistentvolumeclaim/zfssa-block-vs-restore-pvc Bound pvc-91f949f6-5d77-4183-bab5-adfdb1452a90 10Gi RWO zfssa-block-vs-example-sc 16s +``` + +Optionally, verify the new volume exists on ZFS Storage Appliance. Notice that the new +volume is a clone off the snapshot taken from the original volume. + +## Creating pod using restored volume + +Create a pod with the persistent volume claim created from the above step by running the command below: + +```text +kubectl apply -f ../block-snapshot/block-pod-restored-volume.yaml +``` + +The command `kubectl get pod` should now return something similar to this: +```text +NAME READY STATUS RESTARTS AGE +snapshot-controller-0 1/1 Running 0 14d +zfssa-block-vs-example-pod 1/1 Running 0 15m +zfssa-block-vs-restore-pod 1/1 Running 0 21s +zfssa-csi-nodeplugin-7kj5m 2/2 Running 0 16m +zfssa-csi-nodeplugin-rgfzf 2/2 Running 0 16m +zfssa-csi-provisioner-0 4/4 Running 0 16m +``` + +Verify the new volume has the contents of the original volume at the point in time +when the snapsnot was taken. + +```text +kubectl exec -it zfssa-block-vs-restore-pod -- /bin/sh + +/ # cd /dev +/dev # dd if=block bs=64 count=1 +Wed Jan 27 22:06:36 UTC 2021 +1+0 records in +1+0 records out +/dev # +``` + +## Deleting pod, persistent volume claim and volume snapshot + +To delete the pod, persistent volume claim and volume snapshot created from the above steps, +run the following commands below. Wait until the resources being deleted disappear from +the list that `kubectl get ...` command displays before running the next command. + +```text +kubectl delete -f ../block-snapshot/block-pod-restored-volume.yaml +kubectl delete -f ../block-snapshot/block-pvc-from-snapshot.yaml +kubectl delete -f ../block-snapshot/block-snapshot.yaml +``` diff --git a/examples/block-vsc/templates/00-storage-class.yaml b/examples/block-vsc/templates/00-storage-class.yaml new file mode 100644 index 0000000..70d6865 --- /dev/null +++ b/examples/block-vsc/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scBlockName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: "on" + restrictChown: "false" diff --git a/examples/block-vsc/templates/01-pvc.yaml b/examples/block-vsc/templates/01-pvc.yaml new file mode 100644 index 0000000..4fded5a --- /dev/null +++ b/examples/block-vsc/templates/01-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcBlockName }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scBlockName }} diff --git a/examples/block-vsc/templates/02-volume-snapshot-class.yaml b/examples/block-vsc/templates/02-volume-snapshot-class.yaml new file mode 100644 index 0000000..65f7840 --- /dev/null +++ b/examples/block-vsc/templates/02-volume-snapshot-class.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: {{ .Values.vscBlockName }} +driver: zfssa-csi-driver +deletionPolicy: Delete diff --git a/examples/block-vsc/templates/03-pod.yaml b/examples/block-vsc/templates/03-pod.yaml new file mode 100644 index 0000000..2450b52 --- /dev/null +++ b/examples/block-vsc/templates/03-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podBlockName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeDevices: + - name: vol + devicePath: /dev/block + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcBlockName }} diff --git a/examples/block-vsc/values.yaml b/examples/block-vsc/values.yaml new file mode 100644 index 0000000..e45cb18 --- /dev/null +++ b/examples/block-vsc/values.yaml @@ -0,0 +1,19 @@ +# Various names used through example +scBlockName: zfssa-block-example-sc +vscBlockName: zfssa-block-example-vsc +pvcBlockName: zfssa-block-example-pvc +podBlockName: zfssa-block-example-pod + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + +# Settings for volume +volSize: OVERRIDE diff --git a/examples/block/Chart.yaml b/examples/block/Chart.yaml new file mode 100644 index 0000000..2becf53 --- /dev/null +++ b/examples/block/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi-block-example +version: 0.0.1 +description: Deploys an end to end iSCSI volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/block/README.md b/examples/block/README.md new file mode 100644 index 0000000..7c9c45f --- /dev/null +++ b/examples/block/README.md @@ -0,0 +1,63 @@ +# Introduction + +This is an end-to-end example of using iSCSI block devices on a target +Oracle ZFS Storage Appliance. + +Prior to running this example, the iSCSI environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * targetGroup: the target iSCSI group to use on the appliance + * nfsServer: the NFS data path IP address +* volSize: the size of the block volume (iSCSI LUN) to create + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f ../local-values/local-values.yaml zfssa-block ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +``` +kubectl get sc +kubectl get pvc +kubectl get pod +``` + +## Writing data + +Once the pod is deployed, for demo, start the following analytics in a worksheet on +the Oracle ZFS Storage Appliance that is hosting the target LUNs: + +* Protocol -> iSCSI bytes broken down by initiator +* Protocol -> iSCSI bytes broken down by target +* Protocol -> iSCSI bytes broken down by LUN + +Exec into the pod and write some data to the block volume: +```yaml +kubectl exec -it zfssa-block-example-pod -- /bin/sh +/ # cd /dev +/dev # ls +block fd mqueue ptmx random stderr stdout tty zero +core full null pts shm stdin termination-log urandom +/dev # dd if=/dev/zero of=/dev/block count=1024 bs=1024 +1024+0 records in +1024+0 records out +/dev # +``` + +The analytics on the appliance should have seen the spikes as data was written. diff --git a/examples/block/templates/00-storage-class.yaml b/examples/block/templates/00-storage-class.yaml new file mode 100644 index 0000000..70d6865 --- /dev/null +++ b/examples/block/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scBlockName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: "on" + restrictChown: "false" diff --git a/examples/block/templates/01-pvc.yaml b/examples/block/templates/01-pvc.yaml new file mode 100644 index 0000000..4fded5a --- /dev/null +++ b/examples/block/templates/01-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcBlockName }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scBlockName }} diff --git a/examples/block/templates/02-pod.yaml b/examples/block/templates/02-pod.yaml new file mode 100644 index 0000000..2450b52 --- /dev/null +++ b/examples/block/templates/02-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podBlockName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeDevices: + - name: vol + devicePath: /dev/block + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcBlockName }} diff --git a/examples/block/values.yaml b/examples/block/values.yaml new file mode 100644 index 0000000..93138a2 --- /dev/null +++ b/examples/block/values.yaml @@ -0,0 +1,18 @@ +# Various names used through example +scBlockName: zfssa-block-example-sc +pvcBlockName: zfssa-block-example-pvc +podBlockName: zfssa-block-example-pod + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + +# Settings for volume +volSize: OVERRIDE diff --git a/examples/helm/sauron-storage/Chart.yaml b/examples/helm/sauron-storage/Chart.yaml new file mode 100644 index 0000000..3623d47 --- /dev/null +++ b/examples/helm/sauron-storage/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Creates Storageclass and Persistent Volume Claim used by Sauron. +name: sauron-storage +version: 3.0.1 diff --git a/examples/helm/sauron-storage/templates/00-sauron-sc.yaml b/examples/helm/sauron-storage/templates/00-sauron-sc.yaml new file mode 100644 index 0000000..14260df --- /dev/null +++ b/examples/helm/sauron-storage/templates/00-sauron-sc.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.storageClass.volumeType }} + targetGroup: {{ .Values.storageClass.targetGroup }} + blockSize: {{ .Values.storageClass.blockSize }} + pool: {{ .Values.storageClass.pool }} + project: {{ .Values.storageClass.project }} + targetPortal: {{ .Values.storageClass.targetPortal }} + nfsServer: {{ .Values.storageClass.nfsServer }} + rootUser: {{ .Values.storageClass.rootUser }} + rootGroup: {{ .Values.storageClass.rootGroup }} + rootPermissions: {{ .Values.storageClass.rootPermissions }} + shareNFS: {{ .Values.storageClass.shareNFS }} + restrictChown: {{ .Values.storageClass.restrictChown }} diff --git a/examples/helm/sauron-storage/templates/01-sauron-pvc.yaml b/examples/helm/sauron-storage/templates/01-sauron-pvc.yaml new file mode 100644 index 0000000..02ec00d --- /dev/null +++ b/examples/helm/sauron-storage/templates/01-sauron-pvc.yaml @@ -0,0 +1,75 @@ +{{- if .Values.persistentVolumeClaim.enabled -}} +kind: Namespace +apiVersion: v1 +metadata: + name: {{ .Values.persistentVolumeClaim.namespace }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssec0 + namespace: {{ .Values.persistentVolumeClaim.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storageClass.name }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssec1 + namespace: {{ .Values.persistentVolumeClaim.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storageClass.name }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssec2 + namespace: {{ .Values.persistentVolumeClaim.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storageClass.name }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssg + namespace: {{ .Values.persistentVolumeClaim.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storageClass.name }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssp-many + namespace: {{ .Values.persistentVolumeClaim.namespace }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storageClass.name }} +{{- end }} diff --git a/examples/helm/sauron-storage/values.yaml b/examples/helm/sauron-storage/values.yaml new file mode 100644 index 0000000..1635268 --- /dev/null +++ b/examples/helm/sauron-storage/values.yaml @@ -0,0 +1,21 @@ +# Define Storage Class Parameters +storageClass: + name: "sauron-sc" + blockSize: '"8192"' + pool: h1-pool1 + project: pmonday + targetPortal: '"10.80.44.65:3260"' + nfsServer: '"10.80.44.65"' + rootUser: nobody + rootGroup: other + rootPermissions: '"777"' + shareNFS: '"on"' + restrictChown: '"false"' + volumeType: '"thin"' + targetGroup: '"csi-data-path-target"' + +# Define Persistent Volume Claim Parameters. +persistentVolumeClaim: + enabled: true + namespace: sauron + size: 100Gi diff --git a/examples/local-values/README.md b/examples/local-values/README.md new file mode 100644 index 0000000..74a295d --- /dev/null +++ b/examples/local-values/README.md @@ -0,0 +1,6 @@ +# Introduction + +This directory can contain local values files to be used with the helm charts. + +Files in this directory should not be checked in to a source code control +system as they may contain passwords. diff --git a/examples/nfs-exp/Chart.yaml b/examples/nfs-exp/Chart.yaml new file mode 100644 index 0000000..a623b53 --- /dev/null +++ b/examples/nfs-exp/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi-nfs-exp-example +version: 0.0.1 +description: Deploys an end to end NFS volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/nfs-exp/README.md b/examples/nfs-exp/README.md new file mode 100644 index 0000000..19fd467 --- /dev/null +++ b/examples/nfs-exp/README.md @@ -0,0 +1,174 @@ +# Introduction + +This is an end-to-end example of using NFS filesystems and expanding the volume size +on a target Oracle ZFS Storage Appliance. + +Prior to running this example, the NFS environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * nfsServer: the NFS data path IP address +* volSize: the size of the filesystem share to create + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f ../local-values/local-values.yaml zfssa-nfs-exp ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +1. Display the storage class (SC) + The command `kubectl get sc` should now return something similar to this: + + ```text + NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE + zfssa-nfs-exp-example-sc zfssa-csi-driver Delete Immediate true 15s + ``` +2. Display the volume claim + The command `kubectl get pvc` should now return something similar to this: + ```text + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + zfssa-nfs-exp-example-pvc Bound pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 10Gi RWX zfssa-nfs-exp-example-sc 108s + ``` +3. Display the pod mounting the volume + + The command `kubectl get pod` should now return something similar to this: + ```text + NAME READY STATUS RESTARTS AGE + zfssa-csi-nodeplugin-xmv96 2/2 Running 0 43m + zfssa-csi-nodeplugin-z5tmm 2/2 Running 0 43m + zfssa-csi-provisioner-0 4/4 Running 0 43m + zfssa-nfs-exp-example-pod 1/1 Running 0 3m23s + ``` + +## Writing data + +Once the pod is deployed, for demo, start the following analytics in a worksheet on +the Oracle ZFS Storage Appliance that is hosting the target filesystems: + +Exec into the pod and write some data to the NFS volume: +```text +kubectl exec -it zfssa-nfs-exp-example-pod -- /bin/sh + +/ # cd /mnt +/mnt # df -h +Filesystem Size Used Available Use% Mounted on +overlay 38.4G 15.0G 23.4G 39% / +tmpfs 64.0M 0 64.0M 0% /dev +tmpfs 14.6G 0 14.6G 0% /sys/fs/cgroup +shm 64.0M 0 64.0M 0% /dev/shm +tmpfs 14.6G 1.4G 13.2G 9% /tmp/resolv.conf +tmpfs 14.6G 1.4G 13.2G 9% /etc/hostname +:/export/pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 + 10.0G 0 10.0G 0% /mnt +... +/mnt # dd if=/dev/zero of=/mnt/data count=1024 bs=1024 +1024+0 records in +1024+0 records out +/mnt # df -h +Filesystem Size Used Available Use% Mounted on +overlay 38.4G 15.0G 23.4G 39% / +tmpfs 64.0M 0 64.0M 0% /dev +tmpfs 14.6G 0 14.6G 0% /sys/fs/cgroup +shm 64.0M 0 64.0M 0% /dev/shm +tmpfs 14.6G 1.4G 13.2G 9% /tmp/resolv.conf +tmpfs 14.6G 1.4G 13.2G 9% /etc/hostname +:/export/pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 + 10.0G 1.0M 10.0G 0% /mnt + +/mnt # +``` + +The analytics on the appliance should have seen the spikes as data was written. + +## Expanding volume capacity + +After verifying the initially requested capaicy of the NFS volume is provisioned and usable, +exercise expanding of the volume capacity by editing the deployed Persistent Volume Claim. + +Copy ./templates/01-pvc.yaml to /tmp/nfs-exp-pvc.yaml and modify this yaml file for volume expansion, for example: +```text +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: zfssa-nfs-exp-example-pvc +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: "20Gi" + storageClassName: zfssa-nfs-exp-example-sc +``` +Then, apply the updated PVC configuration by running 'kubectl apply -f /tmp/nfs-exp-pvc.yaml' command. Note that the command will return a warning message similar to the following: +```text +Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply +``` + +Alternatively, you can perform volume expansion on the fly using 'kubectl edit' command. +```text +kubectl edit pvc/zfssa-nfs-exp-example-pvc + +... +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: zfssa-nfs-exp-example-sc + volumeMode: Filesystem + volumeName: pvc-27281fde-be45-436d-99a3-b45cddbc74d1 +status: + accessModes: + - ReadWriteMany + capacity: + storage: 10Gi + phase: Bound +... + +Modify the capacity from 10Gi to 20Gi on both spec and status sectioins, then save and exit the edit mode. +``` + +The command `kubectl get pv,pvc` should now return something similar to this: +```text +kubectl get pv,pvc,sc +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +persistentvolume/pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 20Gi RWX Delete Bound default/zfssa-nfs-exp-example-pvc zfssa-nfs-exp-example-sc 129s + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/zfssa-nfs-exp-example-pvc Bound pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 20Gi RWX zfssa-nfs-exp-example-sc 132s +``` + +Exec into the pod and Verify the size of the mounted NFS volume is expanded: +```text +kubectl exec -it zfssa-nfs-exp-example-pod -- /bin/sh + +/ # cd /mnt +/mnt # df -h +Filesystem Size Used Available Use% Mounted on +overlay 38.4G 15.0G 23.4G 39% / +tmpfs 64.0M 0 64.0M 0% /dev +tmpfs 14.6G 0 14.6G 0% /sys/fs/cgroup +shm 64.0M 0 64.0M 0% /dev/shm +tmpfs 14.6G 1.4G 13.2G 9% /tmp/resolv.conf +tmpfs 14.6G 1.4G 13.2G 9% /etc/hostname +:/export/pvc-8325aaa0-bbe3-495b-abb0-0c43cc309624 + 20.0G 1.0M 20.0G 0% /mnt +... +``` diff --git a/examples/nfs-exp/templates/00-storage-class.yaml b/examples/nfs-exp/templates/00-storage-class.yaml new file mode 100644 index 0000000..cc986b6 --- /dev/null +++ b/examples/nfs-exp/templates/00-storage-class.yaml @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scNfsName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: {{ .Values.appliance.shareNFS }} + restrictChown: "false" diff --git a/examples/nfs-exp/templates/01-pvc.yaml b/examples/nfs-exp/templates/01-pvc.yaml new file mode 100644 index 0000000..e04b542 --- /dev/null +++ b/examples/nfs-exp/templates/01-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcNfsName }} +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsName }} diff --git a/examples/nfs-exp/templates/02-pod.yaml b/examples/nfs-exp/templates/02-pod.yaml new file mode 100644 index 0000000..f2976cc --- /dev/null +++ b/examples/nfs-exp/templates/02-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podNfsName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol + mountPath: /mnt + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcNfsName }} + readOnly: false diff --git a/examples/nfs-exp/values.yaml b/examples/nfs-exp/values.yaml new file mode 100644 index 0000000..452839c --- /dev/null +++ b/examples/nfs-exp/values.yaml @@ -0,0 +1,19 @@ +# Various names used through example +scNfsName: zfssa-nfs-exp-example-sc +pvcNfsName: zfssa-nfs-exp-example-pvc +podNfsName: zfssa-nfs-exp-example-pod + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + shareNFS: "on" + +# Settings for volume +volSize: OVERRIDE diff --git a/examples/nfs-multi/Chart.yaml b/examples/nfs-multi/Chart.yaml new file mode 100644 index 0000000..9271d8d --- /dev/null +++ b/examples/nfs-multi/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-nfs-multi-example +version: 0.0.1 +description: Deploys an end to end NFS volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/nfs-multi/README.md b/examples/nfs-multi/README.md new file mode 100644 index 0000000..2934d8b --- /dev/null +++ b/examples/nfs-multi/README.md @@ -0,0 +1,46 @@ +# Introduction + +This is an end-to-end example of using NFS filesystems on a target +Oracle ZFS Storage Appliance. It creates several PVCs and optionally +creates a pod to consume them. + +This example also illustrates the use of namespaces with PVCs and pods. +Be aware that PVCs and pods will be created in the user defined namespace +not in the default namespace as in other examples. + +Prior to running this example, the NFS environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * nfsServer: the NFS data path IP address +* volSize: the size of the filesystem share to create + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f ../local-values/local-values.yaml zfssa-nfs-multi ./ +``` + +## Check pod mounts + +If you enabled the use of the test pod, exec into it and check the NFS volumes: + +``` +kubectl exec -n zfssa-nfs-multi -it zfssa-nfs-multi-example-pod -- /bin/sh +/ # cd /mnt +/mnt # ls +ssec0 ssec1 ssec2 ssg ssp-many +``` diff --git a/examples/nfs-multi/templates/00-storage-class.yaml b/examples/nfs-multi/templates/00-storage-class.yaml new file mode 100644 index 0000000..85ff1fc --- /dev/null +++ b/examples/nfs-multi/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scNfsMultiName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: {{ .Values.appliance.shareNFS }} + restrictChown: "false" diff --git a/examples/nfs-multi/templates/01-pvc.yaml b/examples/nfs-multi/templates/01-pvc.yaml new file mode 100644 index 0000000..31e52ae --- /dev/null +++ b/examples/nfs-multi/templates/01-pvc.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.namespace }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvc0 }} + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsMultiName }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvc1 }} + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsMultiName }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvc2 }} + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsMultiName }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvc3 }} + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsMultiName }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvc4 }} + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsMultiName }} diff --git a/examples/nfs-multi/templates/02-pod.yaml b/examples/nfs-multi/templates/02-pod.yaml new file mode 100644 index 0000000..fc91305 --- /dev/null +++ b/examples/nfs-multi/templates/02-pod.yaml @@ -0,0 +1,49 @@ +{{- if .Values.deployPod -}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podNfsMultiName }} + namespace: {{ .Values.namespace }} + labels: + name: ol7slim-test + +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol0 + mountPath: /mnt/{{ .Values.pvc0 }} + - name: vol1 + mountPath: /mnt/{{ .Values.pvc1 }} + - name: vol2 + mountPath: /mnt/{{ .Values.pvc2 }} + - name: vol3 + mountPath: /mnt/{{ .Values.pvc3 }} + - name: vol4 + mountPath: /mnt/{{ .Values.pvc4 }} + volumes: + - name: vol0 + persistentVolumeClaim: + claimName: {{ .Values.pvc0 }} + readOnly: false + - name: vol1 + persistentVolumeClaim: + claimName: {{ .Values.pvc1 }} + readOnly: false + - name: vol2 + persistentVolumeClaim: + claimName: {{ .Values.pvc2 }} + readOnly: false + - name: vol3 + persistentVolumeClaim: + claimName: {{ .Values.pvc3 }} + readOnly: false + - name: vol4 + persistentVolumeClaim: + claimName: {{ .Values.pvc4 }} + readOnly: false +{{- end }} diff --git a/examples/nfs-multi/values.yaml b/examples/nfs-multi/values.yaml new file mode 100644 index 0000000..007943c --- /dev/null +++ b/examples/nfs-multi/values.yaml @@ -0,0 +1,27 @@ +# Various names used through example +scNfsMultiName: zfssa-nfs-multi-example-sc +pvc0: ssec0 +pvc1: ssec1 +pvc2: ssec2 +pvc3: ssg +pvc4: ssp-many +podNfsMultiName: zfssa-nfs-multi-example-pod +namespace: zfssa-nfs-multi + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + shareNFS: "on" + +# Settings for volume +volSize: OVERRIDE + +# Deploy a pod to consume the volumes +deployPod: true diff --git a/examples/nfs-pv/Chart.yaml b/examples/nfs-pv/Chart.yaml new file mode 100644 index 0000000..b09da80 --- /dev/null +++ b/examples/nfs-pv/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-existing-fs-example +version: 0.0.1 +description: Deploys an end to end filesystem example for an existing Oracle ZFS Storage Appliance CSI filesystem. diff --git a/examples/nfs-pv/README.md b/examples/nfs-pv/README.md new file mode 100644 index 0000000..b997323 --- /dev/null +++ b/examples/nfs-pv/README.md @@ -0,0 +1,91 @@ +# Introduction + +This is an end-to-end example of using an existing filesystem share on a target +Oracle ZFS Storage Appliance. + +Prior to running this example, the NFS environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +This flow to use an existing volume is: +* create a persistent volume (PV) object +* allocate it to a persistent volume claim (PVC) +* use the PVC from a pod + +The following must be set up: +* the volume handle must be a fully formed volume id +* there must be volume attributes defined as part of the persistent volume + +In this example, the volume handle is constructed via values in the helm +chart. The only new attribute necessary is the name of the volume on the +target appliance. The remaining is assembled from the information that is still +in the local-values.yaml file (appliance name, pool, project, etc...). + +The resulting VolumeHandle appears similar to the following, with the values +in ```<>``` filled in from the helm variables: + +``` + volumeHandle: /nfs////local// +``` +From the above, note that the volumeHandle is in the form of an ID with the components: +* 'nfs' - denoting an exported NFS share +* 'appliance name' - this is the management path of the ZFSSA target appliance +* 'volume name' - the name of the share on the appliance +* 'pool name' - the pool on the target appliance +* 'local' - denotes that the pool is owned by the head +* 'project' - the project that the share is in + +In the volume attributes, nfsServer must be defined. + +Once created, a persistent volume claim can be made for this share and used in a pod. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * nfsServer: the NFS data path IP address +* applianceName: the existing appliance name (this is the management path) +* pvExistingFilesystemName: the name of the filesystem share on the target appliance +* volMountPoint: the mount point on the target appliance of the filesystem share +* volSize: the size of the filesystem share + +On the target appliance, ensure that the filesystem share is exported via NFS. + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f ../local-values/local-values.yaml zfssa-nfs-existing ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +``` +kubectl get sc +kubectl get pvc +kubectl get pod +``` + +## Writing data + +Once the pod is deployed, for demo, start the following analytics in a worksheet on +the Oracle ZFS Storage Appliance that is hosting the target filesystems: + +Exec into the pod and write some data to the block volume: +```yaml +kubectl exec -it zfssa-fs-existing-pod -- /bin/sh +/ # cd /mnt +/mnt # ls +/mnt # echo "hello world" > demo.txt +/mnt # +``` + +The analytics on the appliance should have seen the spikes as data was written. diff --git a/examples/nfs-pv/templates/00-storage-class.yaml b/examples/nfs-pv/templates/00-storage-class.yaml new file mode 100644 index 0000000..b6c6482 --- /dev/null +++ b/examples/nfs-pv/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scExistingFilesystemName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: "on" + restrictChown: "false" diff --git a/examples/nfs-pv/templates/01-pv.yaml b/examples/nfs-pv/templates/01-pv.yaml new file mode 100644 index 0000000..dfa9af3 --- /dev/null +++ b/examples/nfs-pv/templates/01-pv.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .Values.pvExistingFilesystemName }} + annotations: + pv.kubernetes.io/provisioned-by: zfssa-csi-driver +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Retain + capacity: + storage: {{ .Values.volSize }} + csi: + driver: zfssa-csi-driver + volumeHandle: /nfs/{{ .Values.applianceName }}/{{ .Values.pvExistingFilesystemName }}/{{ .Values.appliance.pool }}/local/{{ .Values.appliance.project }}/{{ .Values.pvExistingFilesystemName }} + readOnly: false + volumeAttributes: + nfsServer: {{ .Values.appliance.nfsServer }} + share: {{ .Values.volMountPoint }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + rootUser: {{ .Values.appliance.rootUser }} + + claimRef: + namespace: default + name: {{ .Values.pvcExistingFilesystemName }} + diff --git a/examples/nfs-pv/templates/02-pvc.yaml b/examples/nfs-pv/templates/02-pvc.yaml new file mode 100644 index 0000000..976aa68 --- /dev/null +++ b/examples/nfs-pv/templates/02-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcExistingFilesystemName }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scExistingFilesystemName }} diff --git a/examples/nfs-pv/templates/03-pod.yaml b/examples/nfs-pv/templates/03-pod.yaml new file mode 100644 index 0000000..fe25201 --- /dev/null +++ b/examples/nfs-pv/templates/03-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podExistingFilesystemName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol + mountPath: /mnt + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcExistingFilesystemName }} diff --git a/examples/nfs-pv/values.yaml b/examples/nfs-pv/values.yaml new file mode 100644 index 0000000..b9a7971 --- /dev/null +++ b/examples/nfs-pv/values.yaml @@ -0,0 +1,21 @@ +# Various names used through example +scExistingFilesystemName: zfssa-fs-existing-sc +pvExistingFilesystemName: OVERRIDE +pvcExistingFilesystemName: zfssa-fs-existing-pvc +podExistingFilesystemName: zfssa-fs-existing-pod +applianceName: OVERRIDE + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + +# Settings for volume +volMountPoint: OVERRIDE +volSize: OVERRIDE diff --git a/examples/nfs-snapshot/nfs-pod-restored-volume.yaml b/examples/nfs-snapshot/nfs-pod-restored-volume.yaml new file mode 100644 index 0000000..a57feaa --- /dev/null +++ b/examples/nfs-snapshot/nfs-pod-restored-volume.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: zfssa-nfs-vs-restore-pod + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol + mountPath: /mnt + volumes: + - name: vol + persistentVolumeClaim: + claimName: zfssa-nfs-vs-restore-pvc + readOnly: false diff --git a/examples/nfs-snapshot/nfs-pvc-from-snapshot.yaml b/examples/nfs-snapshot/nfs-pvc-from-snapshot.yaml new file mode 100644 index 0000000..079d02b --- /dev/null +++ b/examples/nfs-snapshot/nfs-pvc-from-snapshot.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: zfssa-nfs-vs-restore-pvc +spec: + storageClassName: zfssa-nfs-vs-example-sc + dataSource: + name: zfssa-nfs-vs-snapshot + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 68796 diff --git a/examples/nfs-snapshot/nfs-snapshot.yaml b/examples/nfs-snapshot/nfs-snapshot.yaml new file mode 100644 index 0000000..747f98a --- /dev/null +++ b/examples/nfs-snapshot/nfs-snapshot.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshot +metadata: + name: zfssa-nfs-vs-snapshot +spec: + volumeSnapshotClassName: zfssa-nfs-vs-example-vsc + source: + persistentVolumeClaimName: zfssa-nfs-vs-example-pvc diff --git a/examples/nfs-vsc/Chart.yaml b/examples/nfs-vsc/Chart.yaml new file mode 100644 index 0000000..62f882b --- /dev/null +++ b/examples/nfs-vsc/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi-nfs-vs-example +version: 0.0.1 +description: Deploys an end to end NFS volume snapshot example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/nfs-vsc/README.md b/examples/nfs-vsc/README.md new file mode 100644 index 0000000..c9eabe5 --- /dev/null +++ b/examples/nfs-vsc/README.md @@ -0,0 +1,193 @@ +# Introduction + +This is an end-to-end example of taking a snapshot of an NFS filesystem +volume on a target Oracle ZFS Storage Appliance and making use of it +on another pod by creating (restoring) a volume from the snapshot. + +Prior to running this example, the NFS environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can contain others. The minimum set of values to +customize are: + +* appliance: + * pool: the pool to create shares in + * project: the project to create shares in + * nfsServer: the NFS data path IP address +* volSize: the size of the filesystem share to create + +## Enabling Volume Snapshot Feature (Only for Kubernetes v1.17 - v1.19) + +The Kubernetes Volume Snapshot feature became GA in Kubernetes v1.20. In order to use +this feature in Kubernetes pre-v1.20, it MUST be enabled prior to deploying ZS CSI Driver. +To enable the feature on Kubernetes pre-v1.20, follow the instructions on +[INSTALLATION](../../INSTALLATION.md). + +## Deployment + +This step includes deploying a pod with an NFS volume attached using a regular +storage class and a persistent volume claim. It also deploys a volume snapshot class +required to take snapshots of the persistent volume. + +Assuming there is a set of values in the local-values directory, deploy using Helm 3. If you plan to exercise creating volume from a snapshot with given yaml files as they are, define the names in the local-values.yaml as follows. You can modify them as per your preference. +```text +scNfsName: zfssa-nfs-vs-example-sc +vscNfsName: zfssa-nfs-vs-example-vsc +pvcNfsName: zfssa-nfs-vs-example-pvc +podNfsName: zfssa-nfs-vs-example-pod +``` + +```text +helm ../install -f local-values/local-values.yaml zfssa-nfs-vsc ./ +``` + +Once deployed, verify each of the created entities using kubectl: + +1. Display the storage class (SC) + The command `kubectl get sc` should now return something similar to this: + + ```text + NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE + zfssa-nfs-vs-example-sc zfssa-csi-driver Delete Immediate false 86s + ``` +2. Display the volume claim + The command `kubectl get pvc` should now return something similar to this: + ```text + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + zfssa-nfs-vs-example-pvc Bound pvc-0c1e5351-dc1b-45a4-8f54-b28741d1003e 10Gi RWX zfssa-nfs-vs-example-sc 86s + ``` +3. Display the volume snapshot class + The command `kubectl get volumesnapshotclass` should now return something similar to this: + ```text + NAME DRIVER DELETIONPOLICY AGE + zfssa-nfs-vs-example-vsc zfssa-csi-driver Delete 86s + ``` +4. Display the pod mounting the volume + + The command `kubectl get pod` should now return something similar to this: + ```text + NAME READY STATUS RESTARTS AGE + snapshot-controller-0 1/1 Running 0 6d6h + zfssa-csi-nodeplugin-dx2s4 2/2 Running 0 24m + zfssa-csi-nodeplugin-q9h9w 2/2 Running 0 24m + zfssa-csi-provisioner-0 4/4 Running 0 24m + zfssa-nfs-vs-example-pod 1/1 Running 0 86s + ``` + +## Writing data + +Once the pod is deployed, verify the volume is mounted and can be written. + +```text +kubectl exec -it zfssa-nfs-vs-example-pod -- /bin/sh + +/ # cd /mnt +/mnt # +/mnt # date > timestamp.txt +/mnt # cat timestamp.txt +Tue Jan 19 23:13:10 UTC 2021 +``` + +## Creating snapshot + +Use configuration files in examples/nfs-snapshot directory with proper modifications +for the rest of the example steps. + +Create a snapshot of the volume by running the command below: + +```text +kubectl apply -f ../nfs-snapshot/nfs-snapshot.yaml +``` + +Verify the volume snapshot is created and available by running the following command: + +```text +kubectl get volumesnapshot +``` + +Wait until the READYTOUSE of the snapshot becomes true before moving on to the next steps. +It is important to use the RESTORESIZE value of the volume snapshot just created when specifying +the storage capacity of a persistent volume claim to provision a persistent volume using this +snapshot. For example, the storage capacity in ../nfs-snapshot/nfs-pvc-from-snapshot.yaml + +Optionally, verify the volume snapshot exists on ZFS Storage Appliance. The snapshot name +on ZFS Storage Appliance should have the volume snapshot UID as the suffix. + +## Creating persistent volume claim + +Create a persistent volume claim to provision a volume from the snapshot by running +the command below. Be aware that the persistent volume provisioned by this persistent volume claim +is not expandable. Create a new storage class with allowVolumeExpansion: true and use it when +specifying the persistent volume claim. + + +```text +kubectl apply -f ../nfs-snapshot/nfs-pvc-from-snapshot.yaml +``` + +Verify the persistent volume claim is created and a volume is provisioned by running the following command: + +```text +kubectl get pv,pvc +``` + +The command `kubectl get pv,pvc` should return something similar to this: +```text +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +persistentvolume/pvc-0c1e5351-dc1b-45a4-8f54-b28741d1003e 10Gi RWX Delete Bound default/zfssa-nfs-vs-example-pvc zfssa-nfs-vs-example-sc 34m +persistentvolume/pvc-59d8d447-302d-4438-a751-7271fbbe8238 10Gi RWO Delete Bound default/zfssa-nfs-vs-restore-pvc zfssa-nfs-vs-example-sc 112s + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/zfssa-nfs-vs-example-pvc Bound pvc-0c1e5351-dc1b-45a4-8f54-b28741d1003e 10Gi RWX zfssa-nfs-vs-example-sc 34m +persistentvolumeclaim/zfssa-nfs-vs-restore-pvc Bound pvc-59d8d447-302d-4438-a751-7271fbbe8238 10Gi RWO zfssa-nfs-vs-example-sc 116s +``` + +Optionally, verify the new volume exists on ZFS Storage Appliance. Notice that the new +volume is a clone off the snapshot taken from the original volume. + +## Creating pod using restored volume + +Create a pod with the persistent volume claim created from the above step by running the command below: + +```text +kubectl apply -f ../nfs-snapshot/nfs-pod-restored-volume.yaml +``` + +The command `kubectl get pod` should now return something similar to this: +```text +NAME READY STATUS RESTARTS AGE +snapshot-controller-0 1/1 Running 0 6d7h +zfssa-csi-nodeplugin-dx2s4 2/2 Running 0 68m +zfssa-csi-nodeplugin-q9h9w 2/2 Running 0 68m +zfssa-csi-provisioner-0 4/4 Running 0 68m +zfssa-nfs-vs-example-pod 1/1 Running 0 46m +zfssa-nfs-vs-restore-pod 1/1 Running 0 37s +``` + +Verify the new volume has the contents of the original volume at the point in time +when the snapsnot was taken. + +```text +kubectl exec -it zfssa-nfs-vs-restore-pod -- /bin/sh + +/ # cd /mnt +/mnt # +/mnt # cat timestamp.txt +Tue Jan 19 23:13:10 UTC 2021 +``` + +## Deleting pod, persistent volume claim and volume snapshot + +To delete the pod, persistent volume claim and volume snapshot created from the above steps, +run the following commands below. Wait until the resources being deleted disappear from +the list that `kubectl get ...` command displays before running the next command. + +```text +kubectl delete -f ../nfs-snapshot/nfs-pod-restored-volume.yaml +kubectl delete -f ../nfs-snapshot/nfs-pvc-from-snapshot.yaml +kubectl delete -f ../nfs-snapshot/nfs-snapshot.yaml +``` diff --git a/examples/nfs-vsc/templates/00-storage-class.yaml b/examples/nfs-vsc/templates/00-storage-class.yaml new file mode 100644 index 0000000..d0cc587 --- /dev/null +++ b/examples/nfs-vsc/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scNfsName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: {{ .Values.appliance.shareNFS }} + restrictChown: "false" diff --git a/examples/nfs-vsc/templates/01-pvc.yaml b/examples/nfs-vsc/templates/01-pvc.yaml new file mode 100644 index 0000000..e04b542 --- /dev/null +++ b/examples/nfs-vsc/templates/01-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcNfsName }} +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsName }} diff --git a/examples/nfs-vsc/templates/02-volume-snapshot-class.yaml b/examples/nfs-vsc/templates/02-volume-snapshot-class.yaml new file mode 100644 index 0000000..8cecd2b --- /dev/null +++ b/examples/nfs-vsc/templates/02-volume-snapshot-class.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: {{ .Values.vscNfsName }} +driver: zfssa-csi-driver +deletionPolicy: Delete diff --git a/examples/nfs-vsc/templates/03-pod.yaml b/examples/nfs-vsc/templates/03-pod.yaml new file mode 100644 index 0000000..f2976cc --- /dev/null +++ b/examples/nfs-vsc/templates/03-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podNfsName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol + mountPath: /mnt + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcNfsName }} + readOnly: false diff --git a/examples/nfs-vsc/values.yaml b/examples/nfs-vsc/values.yaml new file mode 100644 index 0000000..d2bd0fe --- /dev/null +++ b/examples/nfs-vsc/values.yaml @@ -0,0 +1,20 @@ +# Various names used through example +scNfsName: zfssa-nfs-vs-example-sc +vscNfsName: zfssa-nfs-vs-example-vsc +pvcNfsName: zfssa-nfs-vs-example-pvc +podNfsName: zfssa-nfs-vs-example-pod + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + shareNFS: "on" + +# Settings for volume +volSize: OVERRIDE diff --git a/examples/nfs/Chart.yaml b/examples/nfs/Chart.yaml new file mode 100644 index 0000000..7bdacac --- /dev/null +++ b/examples/nfs/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: zfssa-csi-nfs-example +version: 0.0.1 +description: Deploys an end to end NFS volume example for Oracle ZFS Storage Appliance CSI driver. diff --git a/examples/nfs/README.md b/examples/nfs/README.md new file mode 100644 index 0000000..964f522 --- /dev/null +++ b/examples/nfs/README.md @@ -0,0 +1,83 @@ +# Introduction + +This is an end-to-end example of using NFS filesystems on a target +Oracle ZFS Storage Appliance. + +Prior to running this example, the NFS environment must be set up properly +on both the Kubernetes worker nodes and the Oracle ZFS Storage Appliance. +Refer to the [INSTALLATION](../../INSTALLATION.md) instructions for details. + +## Configuration + +Set up a local values files. It must contain the values that customize to the +target appliance, but can container others. The minimum set of values to +customize are: + +* appliance: + * targetGroup: the target group that contains data path interfaces on the target appliance + * pool: the pool to create shares in + * project: the project to create shares in + * targetPortal: the target iSCSI portal on the appliance + * nfsServer: the NFS data path IP address +* volSize: the size of the filesystem share to create + +Check out the parameters section of the storage class configuration file (storage-class.yaml) +to see all supporting properties. Refer to NFS Protocol page of Oracle ZFS Storage Appliance +Administration Guide how to defind the values properly. + +## Deployment + +Assuming there is a set of values in the local-values directory, deploy using Helm 3: + +``` +helm install -f local-values/local-values.yaml zfssa-nfs ./nfs +``` + +Once deployed, verify each of the created entities using kubectl: + +1. Display the storage class (SC) + The command `kubectl get sc` should now return something similar to this: + + ```text + NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE + zfssa-csi-nfs-sc zfssa-csi-driver Delete Immediate false 2m9s + ``` +2. Display the volume + The command `kubectl get pvc` should now return something similar to this: + ```text + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + zfssa-csi-nfs-pvc Bound pvc-808d9bd7-cbb0-47a7-b400-b144248f1818 10Gi RWX zfssa-csi-nfs-sc 8s + ``` +3. Display the pod mounting the volume + + The command `kubectl get all` should now return something similar to this: + ```text + NAME READY STATUS RESTARTS AGE + pod/zfssa-csi-nodeplugin-lpts9 2/2 Running 0 25m + pod/zfssa-csi-nodeplugin-vdb44 2/2 Running 0 25m + pod/zfssa-csi-provisioner-0 2/2 Running 0 23m + pod/zfssa-nfs-example-pod 1/1 Running 0 12s + + NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE + daemonset.apps/zfssa-csi-nodeplugin 2 2 2 2 2 25m + + NAME READY AGE + statefulset.apps/zfssa-csi-provisioner 1/1 23m + + ``` + +## Writing data + +Once the pod is deployed, for demo, start the following analytics in a worksheet on +the Oracle ZFS Storage Appliance that is hosting the target filesystems: + +Exec into the pod and write some data to the block volume: +```yaml +kubectl exec -it zfssa-nfs-example-pod -- /bin/sh +/ # cd /mnt +/mnt # ls +/mnt # echo "hello world" > demo.txt +/mnt # +``` + +The analytics on the appliance should have seen the spikes as data was written. diff --git a/examples/nfs/templates/00-storage-class.yaml b/examples/nfs/templates/00-storage-class.yaml new file mode 100644 index 0000000..d0cc587 --- /dev/null +++ b/examples/nfs/templates/00-storage-class.yaml @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.scNfsName }} +provisioner: zfssa-csi-driver +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + volumeType: {{ .Values.appliance.volumeType }} + targetGroup: {{ .Values.appliance.targetGroup }} + blockSize: "8192" + pool: {{ .Values.appliance.pool }} + project: {{ .Values.appliance.project }} + targetPortal: {{ .Values.appliance.targetPortal }} + nfsServer: {{ .Values.appliance.nfsServer }} + rootUser: {{ .Values.appliance.rootUser }} + rootGroup: {{ .Values.appliance.rootGroup }} + rootPermissions: "777" + shareNFS: {{ .Values.appliance.shareNFS }} + restrictChown: "false" diff --git a/examples/nfs/templates/01-pvc.yaml b/examples/nfs/templates/01-pvc.yaml new file mode 100644 index 0000000..e04b542 --- /dev/null +++ b/examples/nfs/templates/01-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcNfsName }} +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: {{ .Values.volSize }} + storageClassName: {{ .Values.scNfsName }} diff --git a/examples/nfs/templates/02-pod.yaml b/examples/nfs/templates/02-pod.yaml new file mode 100644 index 0000000..f2976cc --- /dev/null +++ b/examples/nfs/templates/02-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.podNfsName }} + labels: + name: ol7slim-test +spec: + restartPolicy: Always + containers: + - image: container-registry.oracle.com/os/oraclelinux:7-slim + command: ["/bin/sh", "-c"] + args: [ "tail -f /dev/null" ] + name: ol7slim + volumeMounts: + - name: vol + mountPath: /mnt + volumes: + - name: vol + persistentVolumeClaim: + claimName: {{ .Values.pvcNfsName }} + readOnly: false diff --git a/examples/nfs/values.yaml b/examples/nfs/values.yaml new file mode 100644 index 0000000..b7e182d --- /dev/null +++ b/examples/nfs/values.yaml @@ -0,0 +1,19 @@ +# Various names used through example +scNfsName: zfssa-nfs-example-sc +pvcNfsName: zfssa-nfs-example-pvc +podNfsName: zfssa-nfs-example-pod + +# Settings for target appliance +appliance: + volumeType: thin + targetGroup: OVERRIDE + pool: OVERRIDE + project: OVERRIDE + targetPortal: OVERRIDE + nfsServer: OVERRIDE + rootUser: root + rootGroup: other + shareNFS: "on" + +# Settings for volume +volSize: OVERRIDE diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2bd0b5b --- /dev/null +++ b/go.mod @@ -0,0 +1,53 @@ +module github.com/oracle/zfssa-csi-driver + +go 1.13 + +require ( + github.com/container-storage-interface/spec v1.2.0 + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect + github.com/golang/protobuf v1.4.0 + github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20190415173011-c545557492f4 + github.com/kubernetes-csi/csi-lib-utils v0.6.1 + github.com/onsi/gomega v1.9.0 // indirect + github.com/prometheus/client_golang v1.2.1 // indirect + golang.org/x/net v0.0.0-20191101175033-0deb6923b6d9 + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect + google.golang.org/grpc v1.23.1 + gopkg.in/yaml.v2 v2.2.8 + k8s.io/apimachinery v0.17.11 + k8s.io/client-go v0.18.2 + k8s.io/klog v1.0.0 + k8s.io/kubernetes v1.17.5 + k8s.io/utils v0.0.0-20191114184206-e782cd3c129f +) + +replace ( + k8s.io/api => k8s.io/api v0.17.5 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.5 + k8s.io/apimachinery => k8s.io/apimachinery v0.17.6-beta.0 + k8s.io/apiserver => k8s.io/apiserver v0.17.5 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.5 + k8s.io/client-go => k8s.io/client-go v0.17.5 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.5 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.5 + k8s.io/code-generator => k8s.io/code-generator v0.17.6-beta.0 + k8s.io/component-base => k8s.io/component-base v0.17.5 + k8s.io/cri-api => k8s.io/cri-api v0.17.13-rc.0 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.5 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.5 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.5 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.5 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.5 + k8s.io/kubelet => k8s.io/kubelet v0.17.5 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.5 + k8s.io/metrics => k8s.io/metrics v0.17.5 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.5 +) + +replace k8s.io/kubectl => k8s.io/kubectl v0.17.5 + +replace k8s.io/node-api => k8s.io/node-api v0.17.5 + +replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.17.5 + +replace k8s.io/sample-controller => k8s.io/sample-controller v0.17.5 diff --git a/pkg/service/cluster.go b/pkg/service/cluster.go new file mode 100644 index 0000000..d7d47e3 --- /dev/null +++ b/pkg/service/cluster.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "errors" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + clusterConfig *rest.Config + clientset *kubernetes.Clientset +) + +// Initializes the cluster interface. +// +func InitClusterInterface() error { + + var err error + clusterConfig, err = rest.InClusterConfig() + if err != nil { + fmt.Print("not in cluster mode") + } else { + clientset, err = kubernetes.NewForConfig(clusterConfig) + if err != nil { + return errors.New("could not get Clientset for Kubernetes work") + } + } + + return nil +} + +// Returns the node name based on the passed in node ID. +// +func GetNodeName(nodeID string) (string, error) { + nodeInfo, err := clientset.CoreV1().Nodes().Get(nodeID, metav1.GetOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + ResourceVersion: "1", + }) + + if err != nil { + return "", err + } else { + return nodeInfo.Name, nil + } +} + +// Returns the list of nodes in the form of a slice containing their name. +// +func GetNodeList() ([]string, error) { + + nodeList, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + ResourceVersion: "1", + }) + + if err != nil { + return nil, err + } + + var nodeNameList []string + for _, node:= range nodeList.Items { + nodeNameList = append(nodeNameList, node.Name) + } + + return nodeNameList, nil +} diff --git a/pkg/service/controller.go b/pkg/service/controller.go new file mode 100644 index 0000000..0f9722c --- /dev/null +++ b/pkg/service/controller.go @@ -0,0 +1,688 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "strconv" +) + +var ( + // the current controller service accessModes supported + controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + csi.ControllerServiceCapability_RPC_LIST_VOLUMES, + csi.ControllerServiceCapability_RPC_GET_CAPACITY, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + } +) + +func newZFSSAControllerServer(zd *ZFSSADriver) *csi.ControllerServer { + var cs csi.ControllerServer = zd + return &cs +} + +func (zd *ZFSSADriver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) ( + *csi.CreateVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("CreateVolume", "request", protosanitizer.StripSecrets(req)) + + // Token retrieved + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + // Validate the parameters + if err := validateCreateVolumeReq(ctx, token, req); err != nil { + return nil, err + } + + parameters := req.GetParameters() + pool := parameters["pool"] + project := parameters["project"] + zvol, err := zd.newVolume(ctx, pool, project, + req.GetName(), isBlock(req.GetVolumeCapabilities())) + if err != nil { + return nil, err + } + defer zd.releaseVolume(ctx, zvol) + + if volumeContentSource := req.GetVolumeContentSource(); volumeContentSource != nil { + if snapshot := volumeContentSource.GetSnapshot(); snapshot != nil { + zsnap, err := zd.lookupSnapshot(ctx, token, snapshot.GetSnapshotId()) + if err != nil { + return nil, err + } + defer zd.releaseSnapshot(ctx, zsnap) + return zvol.cloneSnapshot(ctx, token, req, zsnap) + } + return nil, status.Error(codes.InvalidArgument, "Only snapshots are supported as content source") + } else { + return zvol.create(ctx, token, req) + } +} + +// Retrieve the volume size from the request (if not available, use a default) +func getVolumeSize(capRange *csi.CapacityRange) int64 { + volSizeBytes := DefaultVolumeSizeBytes + if capRange != nil { + if capRange.RequiredBytes > 0 { + volSizeBytes = capRange.RequiredBytes + } else if capRange.LimitBytes > 0 && capRange.LimitBytes < volSizeBytes { + volSizeBytes = capRange.LimitBytes + } + } + return volSizeBytes +} + +// Check whether the access mode of the volume to create is "block" or "filesystem" +// +// true block access mode +// false filesystem access mode +// +func isBlock(capabilities []*csi.VolumeCapability) bool { + for _, capacity := range capabilities { + if capacity.GetBlock() == nil { + return false + } + } + return true +} + +// Validates as much of the "create volume request" as possible +// +func validateCreateVolumeReq(ctx context.Context, token *zfssarest.Token, req *csi.CreateVolumeRequest) error { + + log5 := utils.GetLogCTRL(ctx, 5) + + log5.Println("validateCreateVolumeReq started") + + // check the request object is populated + if req == nil { + return status.Errorf(codes.InvalidArgument, "request must not be nil") + } + + reqCaps := req.GetVolumeCapabilities() + if len(reqCaps) == 0 { + return status.Errorf(codes.InvalidArgument, "no accessModes provided") + } + + // check that the name is populated + if req.GetName() == "" { + return status.Error(codes.InvalidArgument, "name must be supplied") + } + + // check as much of the ZFSSA pieces as we can up front, this will cache target information + // in a volatile cache, but in the long run, with many storage classes, this may save us + // quite a few trips to the appliance. Note that different storage classes may have + // different parameters + parameters := req.GetParameters() + poolName, ok := parameters["pool"] + if !ok || len(poolName) < 1 || !utils.IsResourceNameValid(poolName) { + utils.GetLogCTRL(ctx, 3).Println("pool name is invalid", poolName) + return status.Errorf(codes.InvalidArgument, "pool name is invalid (%s)", poolName) + } + + projectName, ok := parameters["project"] + if !ok || len(projectName) < 1 || !utils.IsResourceNameValid(projectName) { + utils.GetLogCTRL(ctx, 3).Println("project name is invalid", projectName) + return status.Errorf(codes.InvalidArgument, "project name is invalid (%s)", projectName) + } + + pool, err := zfssarest.GetPool(ctx, token, poolName) + if err != nil { + return err + } + + if pool.Status != "online" && pool.Status != "degraded" { + log5.Println("Pool not ready", "State", pool.Status) + return status.Errorf(codes.InvalidArgument, "pool %s in an error state (%s)", poolName, pool.Status) + } + + _, err = zfssarest.GetProject(ctx, token, poolName, projectName) + if err != nil { + return err + } + + // If this is a block request, the storage class must have the target group set and it must be on the target + if isBlock(reqCaps) { + err = validateCreateBlockVolumeReq(ctx, token, req) + } else { + err = validateCreateFilesystemVolumeReq(ctx, req) + } + + return err +} + +func (zd *ZFSSADriver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) ( + *csi.DeleteVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("DeleteVolume", + "request", protosanitizer.StripSecrets(req), "context", ctx) + + log2 := utils.GetLogCTRL(ctx, 2) + + // The account to be used for this operation is determined. + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + log2.Println("VolumeID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + zvol, err := zd.lookupVolume(ctx, token, volumeID) + if err != nil { + if status.Convert(err).Code() == codes.NotFound { + log2.Println("Volume already removed", "volume_id", req.GetVolumeId()) + return &csi.DeleteVolumeResponse{}, nil + } else { + log2.Println("Cannot delete volume", "volume_id", req.GetVolumeId(), "error", err.Error()) + return nil, err + } + } + + defer zd.releaseVolume(ctx, zvol) + + entries, err := zvol.getSnapshotsList(ctx, token) + if err != nil { + return nil, err + } + if len(entries) > 0 { + return nil, status.Errorf(codes.FailedPrecondition, "Volume (%s) has snapshots", volumeID) + } + return zvol.delete(ctx, token) +} + +func (zd *ZFSSADriver) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) ( + *csi.ControllerPublishVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("ControllerPublishVolume", + "request", protosanitizer.StripSecrets(req), "volume_context", + req.GetVolumeContext(), "volume_capability", req.GetVolumeCapability()) + + log2 := utils.GetLogCTRL(ctx, 2) + + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + log2.Println("Volume ID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + nodeID := req.GetNodeId() + if len(nodeID) == 0 { + log2.Println("Node ID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Node ID not provided") + } + + capability := req.GetVolumeCapability() + if capability == nil { + log2.Println("Capability not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Capability not provided") + } + + nodeName, err := GetNodeName(nodeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "Node (%s) was not found: %v", req.NodeId, err) + } + + // The account to be used for this operation is determined. + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + zvol, err := zd.lookupVolume(ctx, token, volumeID) + if err != nil { + log2.Println("Volume ID unknown", "volume_id", volumeID, "error", err.Error()) + return nil, err + } + defer zd.releaseVolume(ctx, zvol) + + return zvol.controllerPublishVolume(ctx, token, req, nodeName) +} + +func (zd *ZFSSADriver) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) ( + *csi.ControllerUnpublishVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("ControllerUnpublishVolume", + "request", protosanitizer.StripSecrets(req)) + + log2 := utils.GetLogCTRL(ctx, 2) + + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + log2.Println("Volume ID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + // The account to be used for this operation is determined. + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + zvol, err := zd.lookupVolume(ctx, token, volumeID) + if err != nil { + if status.Convert(err).Code() == codes.NotFound { + log2.Println("Volume already removed", "volume_id", req.GetVolumeId()) + return &csi.ControllerUnpublishVolumeResponse{}, nil + } else { + log2.Println("Cannot unpublish volume", "volume_id", req.GetVolumeId(), "error", err.Error()) + return nil, err + } + } + defer zd.releaseVolume(ctx, zvol) + + return zvol.controllerUnpublishVolume(ctx, token, req) +} + +func (zd *ZFSSADriver) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) ( + *csi.ValidateVolumeCapabilitiesResponse, error) { + + log2 := utils.GetLogCTRL(ctx, 2) + log2.Println("validateVolumeCapabilities", "request", protosanitizer.StripSecrets(req)) + + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "no volume ID provided") + } + + reqCaps := req.GetVolumeCapabilities() + if len(reqCaps) == 0 { + return nil, status.Errorf(codes.InvalidArgument, "no accessModes provided") + } + + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + zvol, err := zd.lookupVolume(ctx, token, volumeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "Volume (%s) was not found: %v", volumeID) + } + defer zd.releaseVolume(ctx, zvol) + + return zvol.validateVolumeCapabilities(ctx, token, req) +} + +func (zd *ZFSSADriver) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) ( + *csi.ListVolumesResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("ListVolumes", "request", protosanitizer.StripSecrets(req)) + + var startIndex int + if len(req.GetStartingToken()) > 0 { + var err error + startIndex, err = strconv.Atoi(req.GetStartingToken()) + if err != nil { + return nil, status.Errorf(codes.Aborted, "invalid starting_token value") + } + } else { + startIndex = 0 + } + + var maxIndex int + maxEntries := int(req.GetMaxEntries()) + if maxEntries < 0 { + return nil, status.Errorf(codes.InvalidArgument, "invalid max_entries value") + } else if maxEntries > 0 { + maxIndex = startIndex + maxEntries + } else { + maxIndex = (1 << 31) - 1 + } + + entries, err := zd.getVolumesList(ctx) + if err != nil { + return nil, err + } + + // The starting index and the maxIndex have to be adjusted based on + // the results of the query. + var nextToken string + + if startIndex >= len(entries) { + // An empty list is returned. + nextToken = "0" + entries = []*csi.ListVolumesResponse_Entry{} + } else if maxIndex >= len(entries) { + // All entries from startIndex are returned. + nextToken = "0" + entries = entries[startIndex:] + } else { + nextToken = strconv.Itoa(maxIndex) + entries = entries[startIndex:maxIndex] + } + + rsp := &csi.ListVolumesResponse{ + NextToken: nextToken, + Entries: entries, + } + + return rsp, nil +} + +func (zd *ZFSSADriver) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) ( + *csi.GetCapacityResponse, error) { + + utils.GetLogCTRL(ctx,5).Println("GetCapacity", "request", protosanitizer.StripSecrets(req)) + + reqCaps := req.GetVolumeCapabilities() + if len(reqCaps) > 0 { + // Providing accessModes is optional, but if provided they must be supported. + var capsValid bool + if isBlock(reqCaps) { + capsValid = areBlockVolumeCapsValid(reqCaps) + } else { + capsValid = areFilesystemVolumeCapsValid(reqCaps) + } + + if !capsValid { + return nil, status.Error(codes.InvalidArgument, "invalid volume accessModes") + } + } + + var availableCapacity int64 + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + parameters := req.GetParameters() + projectName, ok := parameters["project"] + if !ok || len(projectName) == 0 { + // No project name provided the capacity returned will be the capacity + // of the pool if a pool is provided. + poolName, ok := parameters["pool"] + if !ok || len(poolName) == 0 { + // No pool name provided. In this case the sum of the space + // available in each pool is returned. + pools, err := zfssarest.GetPools(ctx, token) + if err != nil { + return nil, err + } + for _, pool := range *pools { + availableCapacity += pool.Usage.Available + } + } else { + // A pool name was provided. The space available in the pool is returned. + pool, err := zfssarest.GetPool(ctx, token, poolName) + if err != nil { + return nil, err + } + availableCapacity = pool.Usage.Available + } + } else { + // A project name was provided. In this case a pool name is required. If + // no pool name was provided, the request is failed. + poolName, ok := parameters["pool"] + if !ok || len(poolName) == 0 { + return nil, status.Error(codes.InvalidArgument, "a pool name is required") + } + project, err := zfssarest.GetProject(ctx, token, poolName, projectName) + if err != nil { + return nil, err + } + availableCapacity = project.SpaceAvailable + } + + return &csi.GetCapacityResponse{AvailableCapacity: availableCapacity}, nil +} + +func (zd *ZFSSADriver) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) ( + *csi.ControllerGetCapabilitiesResponse, error) { + + utils.GetLogCTRL(ctx,5).Println("ControllerGetCapabilities", + "request", protosanitizer.StripSecrets(req)) + + var caps []*csi.ControllerServiceCapability + for _, capacity := range controllerCaps { + c := &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: capacity, + }, + }, + } + caps = append(caps, c) + } + return &csi.ControllerGetCapabilitiesResponse{Capabilities: caps}, nil +} + +func (zd *ZFSSADriver) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) ( + *csi.CreateSnapshotResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("CreateSnapshot", "request", protosanitizer.StripSecrets(req)) + + sourceId := req.GetSourceVolumeId() + snapName := req.GetName() + if len(snapName) == 0 || len(sourceId) == 0 { + return nil, status.Error(codes.InvalidArgument, "Source or snapshot ID missing") + } + + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + zsnap, err := zd.newSnapshot(ctx, token, snapName, sourceId) + if err != nil { + return nil, err + } + defer zd.releaseSnapshot(ctx, zsnap) + + return zsnap.create(ctx, token) +} + +func (zd *ZFSSADriver) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) ( + *csi.DeleteSnapshotResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("DeleteSnapshot", "request", protosanitizer.StripSecrets(req)) + + if len(req.GetSnapshotId()) == 0 { + return nil, status.Errorf(codes.InvalidArgument, "no snapshot ID provided") + } + + log2 := utils.GetLogCTRL(ctx, 2) + + // Retrieve Token + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + // Get exclusive access to the snapshot. + zsnap, err := zd.lookupSnapshot(ctx, token, req.SnapshotId) + if err != nil { + return &csi.DeleteSnapshotResponse{}, nil + } + if err != nil { + if status.Convert(err).Code() == codes.NotFound { + log2.Println("Snapshot already removed", "snapshot_id", req.GetSnapshotId()) + return &csi.DeleteSnapshotResponse{}, nil + } else { + log2.Println("Cannot delete snapshot", "snapshot_id", req.GetSnapshotId(), "error", err.Error()) + return nil, err + } + } + defer zd.releaseSnapshot(ctx, zsnap) + + return zsnap.delete(ctx, token) +} + +func (zd *ZFSSADriver) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) ( + *csi.ListSnapshotsResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("ListSnapshots", "request", protosanitizer.StripSecrets(req)) + + var startIndex int + var err error + if len(req.GetStartingToken()) > 0 { + startIndex, err = strconv.Atoi(req.GetStartingToken()) + if err != nil { + return nil, status.Errorf(codes.Aborted, "invalid starting_token value") + } + } else { + startIndex = 0 + } + + var maxIndex int + maxEntries := int(req.GetMaxEntries()) + if maxEntries < 0 { + return nil, status.Errorf(codes.InvalidArgument, "invalid max_entries value") + } else if maxEntries > 0 { + maxIndex = startIndex + maxEntries + } else { + maxIndex = (1 << 31) - 1 + } + + // Retrieve Token + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + var entries []*csi.ListSnapshotsResponse_Entry + + snapshotId := req.GetSnapshotId() + if len(snapshotId) > 0 { + // Only this snapshot is requested. + zsnap, err := zd.lookupSnapshot(ctx, token, snapshotId) + if err == nil { + entry := new(csi.ListSnapshotsResponse_Entry) + entry.Snapshot = &csi.Snapshot{ + SnapshotId: zsnap.id.String(), + SizeBytes: zsnap.getSize(), + SourceVolumeId: zsnap.getStringSourceId(), + CreationTime: zsnap.getCreationTime(), + ReadyToUse: true, + } + zd.releaseSnapshot(ctx, zsnap) + utils.GetLogCTRL(ctx, 5).Println("ListSnapshots with snapshot ID", "Snapshot", zsnap.getHref()) + entries = append(entries, entry) + } + } else if len(req.GetSourceVolumeId()) > 0 { + // Only snapshots of this volume are requested. + zvol, err := zd.lookupVolume(ctx, token, req.GetSourceVolumeId()) + if err == nil { + entries, err = zvol.getSnapshotsList(ctx, token) + if err != nil { + entries = []*csi.ListSnapshotsResponse_Entry{} + utils.GetLogCTRL(ctx, 5).Println("ListSnapshots with source ID", "Count", len(entries)) + } + zd.releaseVolume(ctx, zvol) + } + } else { + entries, err = zd.getSnapshotList(ctx) + if err != nil { + entries = []*csi.ListSnapshotsResponse_Entry{} + } + utils.GetLogCTRL(ctx, 5).Println("ListSnapshots All", "Count", len(entries)) + } + + // The starting index and the maxIndex have to be adjusted based on + // the results of the query. + var nextToken string + + if startIndex >= len(entries) { + nextToken = "0" + entries = []*csi.ListSnapshotsResponse_Entry{} + } else if maxIndex >= len(entries) { + nextToken = "0" + entries = entries[startIndex:] + } else { + nextToken = strconv.Itoa(maxIndex) + entries = entries[startIndex:maxIndex] + } + + rsp := &csi.ListSnapshotsResponse{ + NextToken: nextToken, + Entries: entries, + } + + return rsp, nil +} + +func (zd *ZFSSADriver) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) ( + *csi.ControllerExpandVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("ControllerExpandVolume", "request", protosanitizer.StripSecrets(req)) + + log2 := utils.GetLogCTRL(ctx, 2) + + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + log2.Println("Volume ID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + zvol, err := zd.lookupVolume(ctx, token, volumeID) + if err != nil { + log2.Println("ControllerExpandVolume request failed, bad VolumeId", + "volume_id", volumeID, "error", err.Error()) + return nil, err + } + defer zd.releaseVolume(ctx, zvol) + + return zvol.controllerExpandVolume(ctx, token, req) +} + +// Check the secrets map (typically in a request context) for a change in the username +// and password or retrieve the username/password from the credentials file, the username +// and password should be scrubbed quickly after use and not remain in memory +func (zd *ZFSSADriver) getUserLogin(ctx context.Context, secrets map[string]string) (string, string, error) { + if secrets != nil { + user, ok := secrets["username"] + if ok { + password := secrets["password"] + return user, password, nil + } + } + + username, err := zd.GetUsernameFromCred() + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("ZFSSA username error:", err) + username = "INVALID_USERNAME" + return "", "", err + } + + password, err := zd.GetPasswordFromCred() + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("ZFSSA password error:", err) + return "", "", err + } + + return username, password, nil +} diff --git a/pkg/service/controller_block.go b/pkg/service/controller_block.go new file mode 100644 index 0000000..e5645c9 --- /dev/null +++ b/pkg/service/controller_block.go @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "context" + "fmt" + "github.com/container-storage-interface/spec/lib/go/csi" + context2 "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "net/http" + "sync/atomic" +) + +// ZFSSA block volume +type zLUN struct { + bolt *utils.Bolt + refcount int32 + state volumeState + href string + id *utils.VolumeId + capacity int64 + accessModes []csi.VolumeCapability_AccessMode + source *csi.VolumeContentSource + initiatorgroup []string + targetgroup string`` +} + +var ( + // access modes supported by block volumes. + blockVolumeCaps = []csi.VolumeCapability_AccessMode { + { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER }, + } +) + +// Creates a new LUN structure. If no information is provided (luninfo is nil), this +// method cannot fail. If information is provided, it will fail if it cannot create +// a volume ID. +func newLUN(vid *utils.VolumeId) *zLUN { + lun := new(zLUN) + lun.id = vid + lun.bolt = utils.NewBolt() + lun.state = stateCreating + return lun +} + +func (lun *zLUN) create(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.create") + + capacityRange := req.GetCapacityRange() + capabilities := req.GetVolumeCapabilities() + + _, luninfo, httpStatus, err := zfssarest.CreateLUN(ctx, token, + req.GetName(), getVolumeSize(capacityRange), &req.Parameters) + if err != nil { + if httpStatus != http.StatusConflict { + lun.state = stateDeleted + return nil, err + } + + utils.GetLogCTRL(ctx, 5).Println("LUN already exits") + // The creation failed because the appliance already has a LUN + // with the same name. We get the information from the appliance, + // update the LUN context and check its compatibility with the request. + if lun.state == stateCreated { + luninfo, _, err := zfssarest.GetLun(ctx, token, + req.Parameters["pool"], req.Parameters["project"], req.GetName()) + if err != nil { + return nil, err + } + lun.setInfo(luninfo) + } + + // The LUN has already been created. The compatibility of the + // capacity range and accessModes is checked. + if !compareCapacityRange(capacityRange, lun.capacity) { + return nil, + status.Errorf(codes.AlreadyExists, + "Volume (%s) is already on target (%s),"+ + " capacity range incompatible (%v), requested (%v/%v)", + lun.id.Name, lun.id.Zfssa, lun.capacity, + capacityRange.RequiredBytes, capacityRange.LimitBytes) + } + if !compareCapabilities(capabilities, lun.accessModes, true) { + return nil, + status.Errorf(codes.AlreadyExists, + "Volume (%s) is already on target (%s), accessModes are incompatible", + lun.id.Name, lun.id.Zfssa) + } + } else { + lun.setInfo(luninfo) + } + + utils.GetLogCTRL(ctx, 5).Printf( + "LUN created: name=%s, target=%s, assigned_number=%d", + luninfo.CanonicalName, luninfo.TargetGroup, luninfo.AssignedNumber[0]) + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: lun.id.String(), + CapacityBytes: lun.capacity, + VolumeContext: req.GetParameters()}}, nil +} + +func (lun *zLUN) cloneSnapshot(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest, zsnap *zSnapshot) (*csi.CreateVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.cloneSnapshot") + + parameters := make(map[string]interface{}) + parameters["project"] = req.Parameters["project"] + parameters["share"] = req.GetName() + parameters["initiatorgroup"] = []string{zfssarest.MaskAll} + + luninfo, _, err := zfssarest.CloneLunSnapshot(ctx, token, zsnap.getHref(), parameters) + if err != nil { + return nil, err + } + + lun.setInfo(luninfo) + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: lun.id.String(), + CapacityBytes: lun.capacity, + VolumeContext: req.GetParameters(), + ContentSource: req.GetVolumeContentSource(), + }}, nil +} + +func (lun *zLUN) delete(ctx context.Context, token *zfssarest.Token) (*csi.DeleteVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.delete") + + if lun.state == stateCreated { + _, httpStatus, err := zfssarest.DeleteLun(ctx, token, lun.id.Pool, lun.id.Project, lun.id.Name) + if err != nil && httpStatus != http.StatusNotFound { + return nil, err + } + + lun.state = stateDeleted + } + + return &csi.DeleteVolumeResponse{}, nil +} + +func (lun *zLUN) controllerPublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerPublishVolumeRequest, nodeName string) (*csi.ControllerPublishVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.controllerPublishVolume") + + pool := lun.id.Pool + project := lun.id.Project + name := lun.id.Name + + list, err := zfssarest.GetInitiatorGroupList(ctx, token, pool, project, name) + if err != nil { + // Log something + return nil, err + } + + // When the driver creates a LUN or clones a Lun from a snapshot of another Lun, + // it masks the intiator group of the Lun using zfssarest.MaskAll value. + // When the driver unpublishes the Lun, it also masks the initiator group. + // This block is to test if the Lun to publish was created or unpublished + // by the driver. Publishing a Lun with unmasked initiator group fails + // to avoid mistakenly publishing a Lun that may be in use by other entity. + utils.GetLogCTRL(ctx, 5).Printf("Volume to publish: %s:%s", lun.id, list[0]) + if len(list) != 1 || list[0] != zfssarest.MaskAll { + var msg string + if len(list) > 0 { + msg = fmt.Sprintf("Volume (%s:%s) may already be published", lun.id, list[0]) + } else { + msg = fmt.Sprintf("Volume (%s) did not return an initiator group list", lun.id) + } + return nil, status.Error(codes.FailedPrecondition, msg) + } + + // Reset the masked initiator group with one named by the current node name. + // There must be initiator groups on ZFSSA defined by the node names. + _, err = zfssarest.SetInitiatorGroupList(ctx, token, pool, project, name, nodeName) + if err != nil { + // Log something + return nil, err + } + + return &csi.ControllerPublishVolumeResponse{}, nil +} + +func (lun *zLUN) controllerUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.controllerUnpublishVolume") + + pool := lun.id.Pool + project := lun.id.Project + name := lun.id.Name + + code, err := zfssarest.SetInitiatorGroupList(ctx, token, pool, project, name, zfssarest.MaskAll) + if err != nil { + utils.GetLogCTRL(ctx, 5).Println("Could not unpublish volume {}, code {}", lun, code) + if code != 404 { + return nil, err + } + utils.GetLogCTRL(ctx, 5).Println("Unpublish failed because LUN was deleted, return success") + } + + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (lun *zLUN) validateVolumeCapabilities(ctx context.Context, token *zfssarest.Token, + req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("lun.validateVolumeCapabilities") + + if areBlockVolumeCapsValid(req.VolumeCapabilities) { + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.VolumeCapabilities, + }, + Message: "", + }, nil + } else { + return &csi.ValidateVolumeCapabilitiesResponse{ + Message: "One or more volume accessModes failed", + }, nil + } +} + +func (lun *zLUN) controllerExpandVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + return nil, status.Error(codes.OutOfRange, "Not allowed for block devices") +} + +func (lun *zLUN) nodeStageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return nil, nil +} + +func (lun *zLUN) nodeUnstageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return nil, nil +} + +func (lun *zLUN) nodePublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + return nil, nil +} + +func (lun *zLUN) nodeUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + return nil, nil +} + +func (lun *zLUN) nodeGetVolumeStats(ctx context.Context, token *zfssarest.Token, + req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, nil +} + +func (lun *zLUN) getDetails(ctx context2.Context, token *zfssarest.Token) (int, error) { + lunInfo, httpStatus, err := zfssarest.GetLun(ctx, token, lun.id.Pool, lun.id.Project, lun.id.Name) + if err != nil { + return httpStatus, err + } + lun.setInfo(lunInfo) + return httpStatus, nil +} + +func (lun *zLUN) getSnapshotsList(ctx context.Context, token *zfssarest.Token) ( + []*csi.ListSnapshotsResponse_Entry, error) { + + snapList, err := zfssarest.GetSnapshots(ctx, token, lun.href) + if err != nil { + return nil, err + } + + return zfssaSnapshotList2csiSnapshotList(ctx, token.Name, snapList), nil +} + +func (lun *zLUN) getState() volumeState { return lun.state } +func (lun *zLUN) getName() string { return lun.id.Name } +func (lun *zLUN) getHref() string { return lun.href } +func (lun *zLUN) getVolumeID() *utils.VolumeId { return lun.id } +func (lun *zLUN) getCapacity() int64 { return lun.capacity } +func (lun *zLUN) isBlock() bool { return true } + +func (lun *zLUN) getSnapshots(ctx context.Context, token *zfssarest.Token) ([]zfssarest.Snapshot, error) { + return zfssarest.GetSnapshots(ctx, token, lun.href) +} + +func (lun *zLUN) setInfo(volInfo interface{}) { + switch luninfo := volInfo.(type) { + case *zfssarest.Lun: + lun.capacity = int64(luninfo.VolumeSize) + lun.href = luninfo.Href + lun.initiatorgroup = luninfo.InitiatorGroup + lun.targetgroup = luninfo.TargetGroup + lun.state = stateCreated + default: + panic("lun.setInfo called with wrong type") + } +} + +// Waits until the file system is available and, when it is, returns with its current state. +func (lun *zLUN) hold(ctx context.Context) volumeState { + utils.GetLogCTRL(ctx, 5).Printf("holding lun (%s)", lun.id.Name) + atomic.AddInt32(&lun.refcount, 1) + return lun.state +} + +// Releases the file system and returns its current reference count. +func (lun *zLUN) release(ctx context.Context) (int32, volumeState) { + utils.GetLogCTRL(ctx, 5).Printf("releasing lun (%s)", lun.id.Name) + return atomic.AddInt32(&lun.refcount, -1), lun.state +} + +func (lun *zLUN) lock(ctx context.Context) volumeState { + utils.GetLogCTRL(ctx, 5).Printf("locking %s", lun.id.String()) + lun.bolt.Lock(ctx) + utils.GetLogCTRL(ctx, 5).Printf("%s is locked", lun.id.String()) + return lun.state +} + +func (lun *zLUN) unlock(ctx context.Context) (int32, volumeState){ + lun.bolt.Unlock(ctx) + utils.GetLogCTRL(ctx, 5).Printf("%s is unlocked", lun.id.String()) + return lun.refcount, lun.state +} + +// Validates the block specific parameters of the create request. +func validateCreateBlockVolumeReq(ctx context.Context, token *zfssarest.Token, req *csi.CreateVolumeRequest) error { + + reqCaps := req.GetVolumeCapabilities() + if !areBlockVolumeCapsValid(reqCaps) { + return status.Error(codes.InvalidArgument, "invalid volume accessModes") + } + + parameters := req.GetParameters() + tg, ok := parameters["targetGroup"] + if !ok || len(tg) < 1 { + return status.Error(codes.InvalidArgument, "a valid ZFSSA target group is required ") + } + + _, err := zfssarest.GetTargetGroup(ctx, token, "iscsi", tg) + if err != nil { + return err + } + + return nil +} + +// Checks whether the capability list is all supported. +func areBlockVolumeCapsValid(volCaps []*csi.VolumeCapability) bool { + + hasSupport := func(cap *csi.VolumeCapability) bool { + for _, c := range blockVolumeCaps { + if c.GetMode() == cap.AccessMode.GetMode() { + return true + } + } + return false + } + + foundAll := true + for _, c := range volCaps { + if !hasSupport(c) { + foundAll = false + break + } + } + + return foundAll +} diff --git a/pkg/service/controller_fs.go b/pkg/service/controller_fs.go new file mode 100644 index 0000000..a927eff --- /dev/null +++ b/pkg/service/controller_fs.go @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "context" + "github.com/container-storage-interface/spec/lib/go/csi" + context2 "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "net/http" + "sync/atomic" +) + +var ( + filesystemAccessModes = []csi.VolumeCapability_AccessMode{ + { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER }, + { Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER }, + { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY }, + { Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY }, + { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY }, + } +) + +// ZFSSA mount volume +type zFilesystem struct { + bolt *utils.Bolt + refcount int32 + state volumeState + href string + id *utils.VolumeId + capacity int64 + accessModes []csi.VolumeCapability_AccessMode + source *csi.VolumeContentSource + mountpoint string +} + +// Creates a new filesysyem structure. If no information is provided (fsinfo is nil), this +// method cannot fail. If information is provided, it will fail if it cannot create a volume ID +func newFilesystem(vid *utils.VolumeId) *zFilesystem { + fs := new(zFilesystem) + fs.id = vid + fs.bolt = utils.NewBolt() + fs.state = stateCreating + return fs +} + +func (fs *zFilesystem) create(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("fs.create") + + capacityRange := req.GetCapacityRange() + capabilities := req.GetVolumeCapabilities() + + fsinfo, httpStatus, err := zfssarest.CreateFilesystem(ctx, token, + req.GetName(), getVolumeSize(capacityRange), &req.Parameters) + if err != nil { + if httpStatus != http.StatusConflict { + fs.state = stateDeleted + return nil, err + } + + utils.GetLogCTRL(ctx, 5).Println("Filesystem already exits") + // The creation failed because the appliance already has a file system + // with the same name. We get the information from the appliance, update + // the file system context and check its compatibility with the request. + if fs.state == stateCreated { + fsinfo, _, err = zfssarest.GetFilesystem(ctx, token, + req.Parameters["pool"], req.Parameters["project"], req.GetName()) + if err != nil { + return nil, err + } + fs.setInfo(fsinfo) + // pass mountpoint as a volume context value to use for nfs mount to the pod + req.Parameters["mountpoint"] = fs.mountpoint + } + + // The volume has already been created. The compatibility of the + // capacity range and accessModes is checked. + if !compareCapacityRange(capacityRange, fs.capacity) { + return nil, + status.Errorf(codes.AlreadyExists, + "Volume (%s) is already on target (%s),"+ + " capacity range incompatible (%v), requested (%v/%v)", + fs.id.Name, fs.id.Zfssa, fs.capacity, + capacityRange.RequiredBytes, capacityRange.LimitBytes) + } + if !compareCapabilities(capabilities, fs.accessModes, false) { + return nil, + status.Errorf(codes.AlreadyExists, + "Volume (%s) is already on target (%s), accessModes are incompatible", + fs.id.Name, fs.id.Zfssa) + } + } else { + fs.setInfo(fsinfo) + // pass mountpoint as a volume context value to use for nfs mount to the pod + req.Parameters["mountpoint"] = fs.mountpoint + } + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: fs.id.String(), + CapacityBytes: fs.capacity, + VolumeContext: req.Parameters}}, nil +} + +func (fs *zFilesystem) cloneSnapshot(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest, zsnap *zSnapshot) (*csi.CreateVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("fs.cloneSnapshot") + + parameters := make(map[string]interface{}) + parameters["project"] = req.Parameters["project"] + parameters["share"] = req.GetName() + + fsinfo, _, err := zfssarest.CloneFileSystemSnapshot(ctx, token, zsnap.getHref(), parameters) + if err != nil { + return nil, err + } + + fs.setInfo(fsinfo) + // pass mountpoint as a volume context value to use for nfs mount to the pod + req.Parameters["mountpoint"] = fs.mountpoint + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: fs.id.String(), + CapacityBytes: fs.capacity, + VolumeContext: req.GetParameters(), + ContentSource: req.GetVolumeContentSource(), + }}, nil +} + +func (fs *zFilesystem) delete(ctx context.Context, token *zfssarest.Token) (*csi.DeleteVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("fs.delete") + + // Check first if the filesystem has snapshots. + snaplist, err := zfssarest.GetSnapshots(ctx, token, fs.href) + if err != nil { + return nil, err + } + + if len(snaplist) > 0 { + return nil, status.Errorf(codes.FailedPrecondition, "filesysytem (%s) has snapshots", fs.id.String()) + } + + _, httpStatus, err := zfssarest.DeleteFilesystem(ctx, token, fs.href) + if err != nil && httpStatus != http.StatusNotFound { + return nil, err + } + + fs.state = stateDeleted + return &csi.DeleteVolumeResponse{}, nil +} + +// Publishes a file system. In this case there's nothing to do. +// +func (fs *zFilesystem) controllerPublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerPublishVolumeRequest, nodeName string) (*csi.ControllerPublishVolumeResponse, error) { + + // Note: the volume context of the volume provisioned from an existing share does not have the mountpoint. + // Use the share (corresponding to volumeAttributes.share of PV configuration) to define the mountpoint. + + return &csi.ControllerPublishVolumeResponse{}, nil +} + + +// Unpublishes a file system. In this case there's nothing to do. +// +func (fs *zFilesystem) controllerUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + utils.GetLogCTRL(ctx, 5).Println("fs.controllerUnpublishVolume") + + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (fs *zFilesystem) validateVolumeCapabilities(ctx context.Context, token *zfssarest.Token, + req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + + if areFilesystemVolumeCapsValid(req.VolumeCapabilities) { + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.VolumeCapabilities, + }, + Message: "", + }, nil + } else { + return &csi.ValidateVolumeCapabilitiesResponse{ + Message: "One or more volume accessModes failed", + }, nil + } +} + +func (fs *zFilesystem) controllerExpandVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("fs.controllerExpandVolume") + + reqCapacity := req.GetCapacityRange().RequiredBytes + if fs.capacity >= reqCapacity { + return &csi.ControllerExpandVolumeResponse{ + CapacityBytes: fs.capacity, + NodeExpansionRequired: false, + }, nil + } + + parameters := make(map[string]interface{}) + parameters["quota"] = reqCapacity + parameters["reservation"] = reqCapacity + fsinfo, _, err := zfssarest.ModifyFilesystem(ctx, token, fs.href, ¶meters) + if err != nil { + return nil, err + } + fs.capacity = fsinfo.Quota + + return &csi.ControllerExpandVolumeResponse{ + CapacityBytes: fsinfo.Quota, + NodeExpansionRequired: false, + }, nil +} + +func (fs *zFilesystem) nodeStageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return nil, nil +} + +func (fs *zFilesystem) nodeUnstageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return nil, nil +} + +func (fs *zFilesystem) nodePublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + return nil, nil +} + +func (fs *zFilesystem) nodeUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + return nil, nil +} + +func (fs *zFilesystem) nodeGetVolumeStats(ctx context.Context, token *zfssarest.Token, + req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, nil +} + +func (fs *zFilesystem) getDetails(ctx context2.Context, token *zfssarest.Token) (int, error) { + fsinfo, httpStatus, err := zfssarest.GetFilesystem(ctx, token, fs.id.Pool, fs.id.Project, fs.id.Name) + if err != nil { + return httpStatus, err + } + fs.setInfo(fsinfo) + return httpStatus, nil +} + +func (fs *zFilesystem) getSnapshotsList(ctx context.Context, token *zfssarest.Token) ( + []*csi.ListSnapshotsResponse_Entry, error) { + + snapList, err := zfssarest.GetSnapshots(ctx, token, fs.href) + if err != nil { + return nil, err + } + return zfssaSnapshotList2csiSnapshotList(ctx, token.Name, snapList), nil +} + +func (fs *zFilesystem) getState() volumeState { return fs.state } +func (fs *zFilesystem) getName() string { return fs.id.Name } +func (fs *zFilesystem) getHref() string { return fs.href } +func (fs *zFilesystem) getVolumeID() *utils.VolumeId { return fs.id } +func (fs *zFilesystem) getCapacity() int64 { return fs.capacity } +func (fs *zFilesystem) isBlock() bool { return false } + +func (fs *zFilesystem) setInfo(volInfo interface{}) { + + switch fsinfo := volInfo.(type) { + case *zfssarest.Filesystem: + fs.capacity = fsinfo.Quota + fs.mountpoint = fsinfo.MountPoint + fs.href = fsinfo.Href + if fsinfo.ReadOnly { + fs.accessModes = filesystemAccessModes[2:len(filesystemAccessModes)] + } else { + fs.accessModes = filesystemAccessModes[0:len(filesystemAccessModes)] + } + fs.state = stateCreated + default: + panic("fs.setInfo called with wrong type") + } +} + +// Waits until the file system is available and, when it is, returns with its current state. +func (fs *zFilesystem) hold(ctx context.Context) volumeState { + utils.GetLogCTRL(ctx, 5).Printf("%s held", fs.id.String()) + atomic.AddInt32(&fs.refcount, 1) + return fs.state +} + +func (fs *zFilesystem) lock(ctx context.Context) volumeState { + utils.GetLogCTRL(ctx, 5).Printf("locking %s", fs.id.String()) + fs.bolt.Lock(ctx) + utils.GetLogCTRL(ctx, 5).Printf("%s is locked", fs.id.String()) + return fs.state +} + +func (fs *zFilesystem) unlock(ctx context.Context) (int32, volumeState){ + fs.bolt.Unlock(ctx) + utils.GetLogCTRL(ctx, 5).Printf("%s is unlocked", fs.id.String()) + return fs.refcount, fs.state +} + +// Releases the file system and returns its current reference count. +func (fs *zFilesystem) release(ctx context.Context) (int32, volumeState) { + utils.GetLogCTRL(ctx, 5).Printf("%s released", fs.id.String()) + return atomic.AddInt32(&fs.refcount, -1), fs.state +} + +// Validates the filesystem specific parameters of the create request. +func validateCreateFilesystemVolumeReq(ctx context.Context, req *csi.CreateVolumeRequest) error { + + reqCaps := req.GetVolumeCapabilities() + if !areFilesystemVolumeCapsValid(reqCaps) { + return status.Error(codes.InvalidArgument, "invalid volume accessModes") + } + return nil +} + +// Checks whether the capability list is all supported. +func areFilesystemVolumeCapsValid(volCaps []*csi.VolumeCapability) bool { + + hasSupport := func(cap *csi.VolumeCapability) bool { + for _, c := range filesystemAccessModes { + if c.GetMode() == cap.AccessMode.GetMode() { + return true + } + } + return false + } + + foundAll := true + for _, c := range volCaps { + if !hasSupport(c) { + foundAll = false + break + } + } + + return foundAll +} diff --git a/pkg/service/identity.go b/pkg/service/identity.go new file mode 100644 index 0000000..4838bb1 --- /dev/null +++ b/pkg/service/identity.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/ptypes/wrappers" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" +) + +func newZFSSAIdentityServer(zd *ZFSSADriver) *csi.IdentityServer { + var id csi.IdentityServer = zd + return &id +} + +func (zd *ZFSSADriver) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) ( + *csi.GetPluginInfoResponse, error) { + + utils.GetLogIDTY(ctx, 5).Println("GetPluginInfo") + + return &csi.GetPluginInfoResponse{ + Name: zd.name, + VendorVersion: zd.version, + }, nil +} + +func (zd *ZFSSADriver) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) ( + *csi.GetPluginCapabilitiesResponse, error) { + + utils.GetLogIDTY(ctx, 5).Println("GetPluginCapabilities") + + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + { + Type: &csi.PluginCapability_VolumeExpansion_{ + VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ + Type: csi.PluginCapability_VolumeExpansion_ONLINE, + }, + }, + }, + }, + }, nil +} + +// This is a readiness probe for the driver, it is for checking if proper drivers are +// loaded. Typical response to failure is a driver restart. +// +func (zd *ZFSSADriver) Probe(ctx context.Context, req *csi.ProbeRequest) ( + *csi.ProbeResponse, error) { + + utils.GetLogIDTY(ctx, 5).Println("Probe") + + // Check that the appliance is responsive, if it is not, we are on hold + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + return nil, grpcStatus.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + _, err = zfssarest.GetServices(ctx, token) + if err != nil { + return &csi.ProbeResponse{ + Ready: &wrappers.BoolValue{Value: false}, + }, grpcStatus.Error(codes.FailedPrecondition, "Failure creating token") + } + + return &csi.ProbeResponse{ + Ready: &wrappers.BoolValue{Value: true}, + }, nil +} diff --git a/pkg/service/iscsi.go b/pkg/service/iscsi.go new file mode 100644 index 0000000..6323c99 --- /dev/null +++ b/pkg/service/iscsi.go @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/container-storage-interface/spec/lib/go/csi" + iscsi_lib "github.com/kubernetes-csi/csi-lib-iscsi/iscsi" + "k8s.io/utils/mount" + "k8s.io/kubernetes/pkg/volume/util" + "os" + "os/exec" + "path" + "strings" +) + +// A subset of the iscsiadm +type IscsiAdmReturnValues int32 + +const( + ISCSI_SUCCESS IscsiAdmReturnValues = 0 + ISCSI_ERR_SESS_NOT_FOUND = 2 + ISCSI_ERR_TRANS_TIMEOUT = 8 + ISCSI_ERR_ISCSID_NOTCONN = 20 + ISCSI_ERR_NO_OBJS_FOUND = 21 +) + +func GetISCSIInfo(ctx context.Context, vid *utils.VolumeId, req *csi.NodePublishVolumeRequest, targetIqn string, + assignedLunNumber int32) (*iscsiDisk, error) { + + volName := vid.Name + tp := req.GetVolumeContext()["targetPortal"] + iqn := targetIqn + + if tp == "" || iqn == "" { + return nil, fmt.Errorf("iSCSI target information is missing (portal=%v), (iqn=%v)", tp, iqn) + } + + portalList := req.GetVolumeContext()["portals"] + if portalList == "" { + portalList = "[]" + } + + utils.GetLogCTRL(ctx, 5).Println("getISCSIInfo", "portal_list", portalList) + secretParams := req.GetVolumeContext()["secret"] + + utils.GetLogCTRL(ctx, 5).Println("getISCSIInfo", "secret_params", secretParams) + secret := parseSecret(secretParams) + sessionSecret, err := parseSessionSecret(secret) + if err != nil { + return nil, err + } + + discoverySecret, err := parseDiscoverySecret(secret) + if err != nil { + return nil, err + } + + utils.GetLogCTRL(ctx, 5).Println("portalMounter", "tp", tp) + portal := portalMounter(tp) + var bkportal []string + bkportal = append(bkportal, portal) + + portals := []string{} + if err := json.Unmarshal([]byte(portalList), &portals); err != nil { + return nil, err + } + + for _, portal := range portals { + bkportal = append(bkportal, portalMounter(string(portal))) + } + + utils.GetLogCTRL(ctx, 5).Println("Built bkportal", "bkportal", bkportal) + iface := req.GetVolumeContext()["iscsiInterface"] + initiatorName := req.GetVolumeContext()["initiatorName"] + chapDiscovery := false + if req.GetVolumeContext()["discoveryCHAPAuth"] == "true" { + chapDiscovery = true + } + + chapSession := false + if req.GetVolumeContext()["sessionCHAPAuth"] == "true" { + chapSession = true + } + + utils.GetLogCTRL(ctx, 5).Println("Final values", "iface", iface, "initiatorName", initiatorName) + i := iscsiDisk{ + VolName: volName, + Portals: bkportal, + Iqn: iqn, + lun: assignedLunNumber, + Iface: iface, + chapDiscovery: chapDiscovery, + chapSession: chapSession, + secret: secret, + sessionSecret: sessionSecret, + discoverySecret: discoverySecret, + InitiatorName: initiatorName, + } + return &i, nil +} + +func GetNodeISCSIInfo(vid *utils.VolumeId, req *csi.NodePublishVolumeRequest, targetIqn string, assignedLunNumber int32) ( + *iscsiDisk, error) { + + volName := vid.Name + tp := req.GetVolumeContext()["targetPortal"] + iqn := targetIqn + if tp == "" || iqn == "" { + return nil, fmt.Errorf("iSCSI target information is missing") + } + + portalList := req.GetVolumeContext()["portals"] + if portalList == "" { + portalList = "[]" + } + secretParams := req.GetVolumeContext()["secret"] + secret := parseSecret(secretParams) + sessionSecret, err := parseSessionSecret(secret) + if err != nil { + return nil, err + } + + discoverySecret, err := parseDiscoverySecret(secret) + if err != nil { + return nil, err + } + + // For ZFSSA, the portal should also contain the assigned number + portal := portalMounter(tp) + var bkportal []string + bkportal = append(bkportal, portal) + + portals := []string{} + if err := json.Unmarshal([]byte(portalList), &portals); err != nil { + return nil, err + } + + for _, portal := range portals { + bkportal = append(bkportal, portalMounter(string(portal))) + } + + iface := req.GetVolumeContext()["iscsiInterface"] + initiatorName := req.GetVolumeContext()["initiatorName"] + chapDiscovery := false + if req.GetVolumeContext()["discoveryCHAPAuth"] == "true" { + chapDiscovery = true + } + + chapSession := false + if req.GetVolumeContext()["sessionCHAPAuth"] == "true" { + chapSession = true + } + + i := iscsiDisk{ + VolName: volName, + Portals: bkportal, + Iqn: iqn, + lun: assignedLunNumber, + Iface: iface, + chapDiscovery: chapDiscovery, + chapSession: chapSession, + secret: secret, + sessionSecret: sessionSecret, + discoverySecret: discoverySecret, + InitiatorName: initiatorName, + } + return &i, nil +} + +func buildISCSIConnector(iscsiInfo *iscsiDisk) *iscsi_lib.Connector { + c := iscsi_lib.Connector{ + VolumeName: iscsiInfo.VolName, + TargetIqn: iscsiInfo.Iqn, + TargetPortals: iscsiInfo.Portals, + Lun: iscsiInfo.lun, + Multipath: len(iscsiInfo.Portals) > 1, + } + + if iscsiInfo.sessionSecret != (iscsi_lib.Secrets{}) { + c.SessionSecrets = iscsiInfo.sessionSecret + if iscsiInfo.discoverySecret != (iscsi_lib.Secrets{}) { + c.DiscoverySecrets = iscsiInfo.discoverySecret + } + } + + return &c +} + +func GetISCSIDiskMounter(iscsiInfo *iscsiDisk, readOnly bool, fsType string, mountOptions []string, + targetPath string) *iscsiDiskMounter { + + return &iscsiDiskMounter{ + iscsiDisk: iscsiInfo, + fsType: fsType, + readOnly: readOnly, + mountOptions: mountOptions, + mounter: &mount.SafeFormatAndMount{Interface: mount.New("")}, + targetPath: targetPath, + deviceUtil: util.NewDeviceHandler(util.NewIOHandler()), + connector: buildISCSIConnector(iscsiInfo), + } +} + +func GetISCSIDiskUnmounter(volumeId *utils.VolumeId) *iscsiDiskUnmounter { + volName := volumeId.Name + return &iscsiDiskUnmounter{ + iscsiDisk: &iscsiDisk{ + VolName: volName, + }, + mounter: mount.New(""), + } +} + +func portalMounter(portal string) string { + if !strings.Contains(portal, ":") { + portal = portal + ":3260" + } + return portal +} + +func parseSecret(secretParams string) map[string]string { + var secret map[string]string + if err := json.Unmarshal([]byte(secretParams), &secret); err != nil { + return nil + } + return secret +} + +func parseSessionSecret(secretParams map[string]string) (iscsi_lib.Secrets, error) { + var ok bool + secret := iscsi_lib.Secrets{} + + if len(secretParams) == 0 { + return secret, nil + } + + if secret.UserName, ok = secretParams["node.session.auth.username"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.session.auth.username not found in secret") + } + if secret.Password, ok = secretParams["node.session.auth.password"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.session.auth.password not found in secret") + } + if secret.UserNameIn, ok = secretParams["node.session.auth.username_in"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.session.auth.username_in not found in secret") + } + if secret.PasswordIn, ok = secretParams["node.session.auth.password_in"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.session.auth.password_in not found in secret") + } + + secret.SecretsType = "chap" + return secret, nil +} + +func parseDiscoverySecret(secretParams map[string]string) (iscsi_lib.Secrets, error) { + var ok bool + secret := iscsi_lib.Secrets{} + + if len(secretParams) == 0 { + return secret, nil + } + + if secret.UserName, ok = secretParams["node.sendtargets.auth.username"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.sendtargets.auth.username not found in secret") + } + if secret.Password, ok = secretParams["node.sendtargets.auth.password"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.sendtargets.auth.password not found in secret") + } + if secret.UserNameIn, ok = secretParams["node.sendtargets.auth.username_in"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.sendtargets.auth.username_in not found in secret") + } + if secret.PasswordIn, ok = secretParams["node.sendtargets.auth.password_in"]; !ok { + return iscsi_lib.Secrets{}, fmt.Errorf("node.sendtargets.auth.password_in not found in secret") + } + + secret.SecretsType = "chap" + return secret, nil +} + +type iscsiDisk struct { + Portals []string + Iqn string + lun int32 + Iface string + chapDiscovery bool + chapSession bool + secret map[string]string + sessionSecret iscsi_lib.Secrets + discoverySecret iscsi_lib.Secrets + InitiatorName string + VolName string +} + +type iscsiDiskMounter struct { + *iscsiDisk + readOnly bool + fsType string + mountOptions []string + mounter *mount.SafeFormatAndMount + deviceUtil util.DeviceUtil + targetPath string + connector *iscsi_lib.Connector +} + +type iscsiDiskUnmounter struct { + *iscsiDisk + mounter mount.Interface +} + + +type ISCSIUtil struct{} + +func (util *ISCSIUtil) Rescan (ctx context.Context) (string, error) { + cmd := exec.Command("iscsiadm", "-m", "session", "--rescan") + var stdout bytes.Buffer + var iscsiadmError error + cmd.Stdout = &stdout + cmd.Stderr = &stdout + defer stdout.Reset() + + // we're using Start and Wait because we want to grab exit codes + err := cmd.Start() + if err != nil { + // Check if this is simply no sessions found (rc 21) + exitCode := err.(*exec.ExitError).ExitCode() + if exitCode == ISCSI_ERR_NO_OBJS_FOUND { + // No error, just no objects found + utils.GetLogUTIL(ctx, 4).Println("iscsiadm: no sessions, will continue (start path)") + } else { + formattedOutput := strings.Replace(string(stdout.Bytes()), "\n", "", -1) + iscsiadmError = fmt.Errorf("iscsiadm error: %s (%s)", formattedOutput, err.Error()) + } + return string(stdout.Bytes()), iscsiadmError + } + + err = cmd.Wait() + if err != nil { + exitCode := err.(*exec.ExitError).ExitCode() + if exitCode == ISCSI_ERR_NO_OBJS_FOUND { + // No error, just no objects found + utils.GetLogUTIL(ctx, 4).Println("iscsiadm: no sessions, will continue (wait path)") + } else { + formattedOutput := strings.Replace(string(stdout.Bytes()), "\n", "", -1) + iscsiadmError = fmt.Errorf("iscsiadm error: %s (%s)", formattedOutput, err.Error()) + } + } + return string(stdout.Bytes()), iscsiadmError +} + +func (util *ISCSIUtil) ConnectDisk(ctx context.Context, b iscsiDiskMounter) (string, error) { + utils.GetLogUTIL(ctx, 4).Println("ConnectDisk started") + _, err := util.Rescan(ctx) + if err != nil { + utils.GetLogUTIL(ctx, 4).Println("iSCSI rescan error: %s", err.Error()) + return "", err + } + utils.GetLogUTIL(ctx, 4).Println("ConnectDisk will connect and get device path") + devicePath, err := iscsi_lib.Connect(*b.connector) + if err != nil { + utils.GetLogUTIL(ctx, 4).Println("iscsi_lib connect error: %s", err.Error()) + return "", err + } + + if devicePath == "" { + utils.GetLogUTIL(ctx, 4).Println("iscsi_lib devicePath is empty, cannot continue") + return "", fmt.Errorf("connect reported success, but no path returned") + } + utils.GetLogUTIL(ctx, 4).Println("ConnectDisk devicePath: %s", devicePath) + return devicePath, nil +} + +func (util *ISCSIUtil) AttachDisk(ctx context.Context, b iscsiDiskMounter, devicePath string) (string, error) { + // Mount device + if len(devicePath) == 0 { + localDevicePath, err := util.ConnectDisk(ctx, b) + if err != nil { + utils.GetLogUTIL(ctx, 3).Println("ConnectDisk failure: %s", err.Error()) + return "", err + } + devicePath = localDevicePath + } + + mntPath := b.targetPath + notMnt, err := b.mounter.IsLikelyNotMountPoint(mntPath) + if err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("heuristic determination of mount point failed: %v", err) + } + if !notMnt { + utils.GetLogUTIL(ctx, 3).Println("iscsi: device already mounted", "mount_path", mntPath) + return "", nil + } + + if err := os.MkdirAll(mntPath, 0750); err != nil { + return "", err + } + + // Persist iscsi disk config to json file for DetachDisk path + file := path.Join(mntPath, b.VolName+".json") + err = iscsi_lib.PersistConnector(b.connector, file) + if err != nil { + return "", err + } + + options := []string{"bind"} + + if b.readOnly { + options = append(options, "ro") + } else { + options = append(options, "rw") + } + options = append(options, b.mountOptions...) + + utils.GetLogUTIL(ctx, 3).Println("Mounting disk at path: %s", mntPath) + err = b.mounter.Mount(devicePath, mntPath, "", options) + if err != nil { + utils.GetLogUTIL(ctx, 3).Println("iscsi: failed to mount iscsi volume", + "device_path", devicePath, "mount_path", mntPath, "error", err.Error()) + return "", err + } + + return devicePath, err +} + +func (util *ISCSIUtil) DetachDisk(ctx context.Context, c iscsiDiskUnmounter, targetPath string) error { + _, cnt, err := mount.GetDeviceNameFromMount(c.mounter, targetPath) + if err != nil { + return err + } + + if pathExists, pathErr := mount.PathExists(targetPath); pathErr != nil { + return fmt.Errorf("Error checking if path exists: %v", pathErr) + } else if !pathExists { + utils.GetLogUTIL(ctx, 2).Println("Unmount skipped because path does not exist", + "target_path", targetPath) + return nil + } + if err = c.mounter.Unmount(targetPath); err != nil { + utils.GetLogUTIL(ctx, 3).Println("iscsi detach disk: failed to unmount", + "target_path", targetPath, "error", err.Error()) + return err + } + + cnt-- + if cnt != 0 { + return nil + } + + // load iscsi disk config from json file + file := path.Join(targetPath, c.iscsiDisk.VolName+".json") + connector, err := iscsi_lib.GetConnectorFromFile(file) + if err != nil { + return err + } + + err = iscsi_lib.Disconnect(connector.TargetIqn, connector.TargetPortals) + if err := os.RemoveAll(targetPath); err != nil { + return err + } + + return nil +} diff --git a/pkg/service/mount.go b/pkg/service/mount.go new file mode 100644 index 0000000..208021d --- /dev/null +++ b/pkg/service/mount.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "k8s.io/utils/mount" + "os" +) + +// Mounter is an interface for mount operations +type Mounter interface { + mount.Interface + GetDeviceName(mountPath string) (string, int, error) + MakeFile(pathname string) error + ExistsPath(pathname string) (bool, error) +} + +type NodeMounter struct { + mount.SafeFormatAndMount +} + +func newNodeMounter() Mounter { + return &NodeMounter{ + mount.SafeFormatAndMount{ + Interface: mount.New(""), + }, + } +} + +// Retrieve a device name from a mount point (this is a compatibility interface) +func (m *NodeMounter) GetDeviceName(mountPath string) (string, int, error) { + return mount.GetDeviceNameFromMount(m, mountPath) +} + +// Make a file at the pathname +func (mounter *NodeMounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + + if err != nil && !os.IsExist(err) { + return err + } + + return nil +} + +// Check if a file exists +func (mount *NodeMounter) ExistsPath(pathname string) (bool, error) { + // Check if the global mount path exists and create it if it does not + exists := true + _, err := os.Stat(pathname) + + if _, err := os.Stat(pathname); os.IsNotExist(err) { + exists = false + } + + return exists, err +} diff --git a/pkg/service/node.go b/pkg/service/node.go new file mode 100644 index 0000000..f1d6ef8 --- /dev/null +++ b/pkg/service/node.go @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "fmt" + "os" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + // nodeCaps represents the capability of node service. + nodeCaps = []csi.NodeServiceCapability_RPC_Type{ + csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, +// csi.NodeServiceCapability_RPC_EXPAND_VOLUME, + csi.NodeServiceCapability_RPC_UNKNOWN, + } +) + +func NewZFSSANodeServer(zd *ZFSSADriver) *csi.NodeServer { + zd.NodeMounter = newNodeMounter() + var ns csi.NodeServer = zd + return &ns +} + +func (zd *ZFSSADriver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeStageVolume", "request", req) + + // The request validity of the request is checked + VolumeID := req.GetVolumeId() + if len(VolumeID) == 0 { + utils.GetLogNODE(ctx, 2).Println("VolumeID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + targetPath := req.GetStagingTargetPath() + if len(targetPath) == 0 { + utils.GetLogNODE(ctx, 2).Println("Target path not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + reqCaps := req.GetVolumeCapability() + if reqCaps == nil { + utils.GetLogNODE(ctx, 2).Println("Capability not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Capability not provided") + } + + // Not staging for either block or mount for now. + return &csi.NodeStageVolumeResponse{}, nil +} + +func (zd *ZFSSADriver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeUnStageVolume", "request", req) + + VolumeID := req.GetVolumeId() + if len(VolumeID) == 0 { + utils.GetLogNODE(ctx, 2).Println("VolumeID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + target := req.GetStagingTargetPath() + if len(target) == 0 { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + + // Check if target directory is a mount point. GetDeviceNameFromMount + // given a mnt point, finds the device from /proc/mounts + // returns the device name, reference count, and error code + dev, refCount, err := zd.NodeMounter.GetDeviceName(target) + if err != nil { + msg := fmt.Sprintf("failed to check if volume is mounted: %v", err) + return nil, status.Error(codes.Internal, msg) + } + + // From the spec: If the volume corresponding to the volume_id + // is not staged to the staging_target_path, the Plugin MUST + // reply 0 OK. + if refCount == 0 { + utils.GetLogNODE(ctx, 3).Println("NodeUnstageVolume: target not mounted", "target", target) + return &csi.NodeUnstageVolumeResponse{}, nil + } + + if refCount > 1 { + utils.GetLogNODE(ctx, 2).Println("NodeUnstageVolume: found references to device mounted at target path", + "references", refCount, "device", dev, "target", target) + } + + utils.GetLogNODE(ctx, 5).Println("NodeUnstageVolume: unmounting target", "target", target) + err = zd.NodeMounter.Unmount(target) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot unmount staging target %q: %v", target, err) + } + + notMnt, mntErr := zd.NodeMounter.IsLikelyNotMountPoint(target) + if mntErr != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot determine staging target path", + "staging_target_path", target, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + if notMnt { + if err := os.Remove(target); err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot delete staging target path", + "staging_target_path", target, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (zd *ZFSSADriver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) ( + *csi.NodePublishVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodePublishVolume", "request", req) + + VolumeID := req.GetVolumeId() + if len(VolumeID) == 0 { + utils.GetLogNODE(ctx, 2).Println("VolumeID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + zVolumeId, err := utils.VolumeIdFromString(VolumeID) + if err != nil { + utils.GetLogNODE(ctx, 2).Println("NodePublishVolume Volume ID was invalid", + "volume_id", req.GetVolumeId(), "error", err.Error()) + // NOTE: by spec, we should return success since there is nothing to delete + return nil, status.Error(codes.InvalidArgument, "Volume ID invalid") + } + + source := req.GetStagingTargetPath() + if len(source) == 0 { + utils.GetLogNODE(ctx, 2).Println("Staging target path not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + + target := req.GetTargetPath() + if len(target) == 0 { + utils.GetLogNODE(ctx, 2).Println("Target path not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + utils.GetLogNODE(ctx, 2).Printf("NodePublishVolume: stagingTarget=%s, target=%s", source, target) + + volCap := req.GetVolumeCapability() + if volCap == nil { + utils.GetLogNODE(ctx, 2).Println("Volume Capabilities path not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume capability not provided") + } + + // The account to be used for this operation is determined. + user, password, err := zd.getUserLogin(ctx, req.Secrets) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + + var mountOptions []string + if req.GetReadonly() { + mountOptions = append(mountOptions, "ro") + } + + if req.GetVolumeCapability().GetBlock() != nil { + mountOptions = append(mountOptions, "bind") + return zd.nodePublishBlockVolume(ctx, token, req, zVolumeId, mountOptions) + } + + switch mode := volCap.GetAccessType().(type) { + case *csi.VolumeCapability_Block: + mountOptions = append(mountOptions, "bind") + return zd.nodePublishBlockVolume(ctx, token, req, zVolumeId, mountOptions) + case *csi.VolumeCapability_Mount: + return zd.nodePublishFileSystem(ctx, token, req, zVolumeId, mountOptions, mode) + default: + utils.GetLogNODE(ctx, 2).Println("Publish does not support Access Type", "access_type", + volCap.GetAccessType()) + return nil, status.Error(codes.InvalidArgument, "Invalid access type") + } +} + +func (zd *ZFSSADriver) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) ( + *csi.NodeUnpublishVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeUnpublishVolume", "request", req) + + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + utils.GetLogNODE(ctx, 2).Println("VolumeID not provided, will return") + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + zVolumeId, err := utils.VolumeIdFromString(volumeID) + if err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot unpublish volume", + "volume_id", req.GetVolumeId(), "error", err.Error()) + return nil, err + } + + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "Invalid credentials") + } + token := zfssarest.LookUpToken(user, password) + if zVolumeId.IsBlock() { + return zd.nodeUnpublishBlockVolume(ctx, token, req, zVolumeId) + } else { + return zd.nodeUnpublishFilesystemVolume(token, ctx, req, zVolumeId) + } +} + +func (zd *ZFSSADriver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) ( + *csi.NodeGetVolumeStatsResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeGetVolumeStats", "request", req) + + return nil, status.Error(codes.Unimplemented, "") +} + +func (zd *ZFSSADriver) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) ( + *csi.NodeExpandVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeExpandVolume", "request", req) + + return nil, status.Error(codes.Unimplemented, "") +} + +func (zd *ZFSSADriver) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) ( + *csi.NodeGetCapabilitiesResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeGetCapabilities", "request", req) + + var caps []*csi.NodeServiceCapability + for _, capacity := range nodeCaps { + c := &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: capacity, + }, + }, + } + caps = append(caps, c) + } + return &csi.NodeGetCapabilitiesResponse{Capabilities: caps}, nil + +} + +func (zd *ZFSSADriver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) ( + *csi.NodeGetInfoResponse, error) { + + utils.GetLogNODE(ctx, 2).Println("NodeGetInfo", "request", req) + + return &csi.NodeGetInfoResponse{ + NodeId: zd.config.NodeName, + }, nil +} diff --git a/pkg/service/node_block.go b/pkg/service/node_block.go new file mode 100644 index 0000000..2d21c6d --- /dev/null +++ b/pkg/service/node_block.go @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Nothing is done +func (zd *ZFSSADriver) NodeStageBlockVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + + return &csi.NodeStageVolumeResponse{}, nil +} + +func (zd *ZFSSADriver) NodeUnstageBlockVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("NodeUnStageVolume", "request", req, "context", ctx) + + target := req.GetStagingTargetPath() + if len(target) == 0 { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + + // Check if target directory is a mount point. GetDeviceNameFromMount + // given a mnt point, finds the device from /proc/mounts + // returns the device name, reference count, and error code + dev, refCount, err := zd.NodeMounter.GetDeviceName(target) + if err != nil { + msg := fmt.Sprintf("failed to check if volume is mounted: %v", err) + return nil, status.Error(codes.Internal, msg) + } + + // From the spec: If the volume corresponding to the volume_id + // is not staged to the staging_target_path, the Plugin MUST + // reply 0 OK. + if refCount == 0 { + utils.GetLogNODE(ctx, 3).Println("NodeUnstageVolume: target not mounted", "target", target) + return &csi.NodeUnstageVolumeResponse{}, nil + } + + if refCount > 1 { + utils.GetLogNODE(ctx, 2).Println("NodeUnstageVolume: found references to device mounted at target path", + "references", refCount, "device", dev, "target", target) + } + + utils.GetLogNODE(ctx, 5).Println("NodeUnstageVolume: unmounting target", "target", target) + err = zd.NodeMounter.Unmount(target) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not unmount target %q: %v", target, err) + } + + return &csi.NodeUnstageVolumeResponse{}, nil +} + +// nodePublishBlockVolume is the worker for block volumes only, it is going to get the +// block device mounted to the target path so it can be moved to the container requesting it +func (zd *ZFSSADriver) nodePublishBlockVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodePublishVolumeRequest, vid *utils.VolumeId, mountOptions []string) ( + *csi.NodePublishVolumeResponse, error) { + + target := req.GetTargetPath() + + utils.GetLogNODE(ctx, 5).Println("nodePublishBlockVolume", req) + devicePath, err := attachBlockVolume(ctx, token, req, vid) + if err != nil { + return nil, err + } + utils.GetLogNODE(ctx, 5).Println("nodePublishBlockVolume", "devicePath", devicePath) + + _, err = zd.NodeMounter.ExistsPath(devicePath) + if err != nil { + return nil, err + } + + globalMountPath := filepath.Dir(target) + + // create the global mount path if it is missing + // Path in the form of /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/{volumeName} + utils.GetLogNODE(ctx, 5).Println("NodePublishVolume [block]", "globalMountPath", globalMountPath) + + // Check if the global mount path exists and create it if it does not + if _, err := os.Stat(globalMountPath); os.IsNotExist(err) { + err := os.Mkdir(globalMountPath, 0700) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not create dir %q: %v", globalMountPath, err) + } + } else if err != nil { + return nil, status.Errorf(codes.Internal, "Could not check if path exists %q: %v", globalMountPath, err) + } + + utils.GetLogNODE(ctx, 5).Println("NodePublishVolume [block]: making target file", "target_file", target) + + // Create a new target file for the mount location + err = zd.NodeMounter.MakeFile(target) + if err != nil { + if removeErr := os.Remove(target); removeErr != nil { + return nil, status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr) + } + return nil, status.Errorf(codes.Internal, "Could not create file %q: %v", target, err) + } + + utils.GetLogNODE(ctx, 5).Println("NodePublishVolume [block]: mounting block device", + "device_path", devicePath, "target", target, "mount_options", mountOptions) + if err := zd.NodeMounter.Mount(devicePath, target, "", mountOptions); err != nil { + if removeErr := os.Remove(target); removeErr != nil { + return nil, status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr) + } + return nil, status.Errorf(codes.Internal, "Could not mount %q at %q: %v", devicePath, target, err) + } + utils.GetLogNODE(ctx, 5).Println("NodePublishVolume [block]: mounted block device", + "device_path", devicePath, "target", target) + return &csi.NodePublishVolumeResponse{}, nil +} + +// attachBlockVolume rescans the iSCSI session and attempts to attach the disk. +// This may actually belong in ControllerPublish (and was there for a while), but +// if it goes in the controller, then we have to have a way to remote the request +// to the proper node since the controller may not co-exist with the node where +// the device is actually needed for the work to be done +func attachBlockVolume(ctx context.Context, token *zfssarest.Token, req *csi.NodePublishVolumeRequest, + vid *utils.VolumeId) (string, error) { + + lun := vid.Name + pool := vid.Pool + project := vid.Project + + lunInfo, _, err := zfssarest.GetLun(nil, token, pool, project, lun) + if err != nil { + return "", err + } + + targetGroup := lunInfo.TargetGroup + targetInfo, err := zfssarest.GetTargetGroup(nil, token, "iscsi", targetGroup) + if err != nil { + return "", err + } + + iscsiInfo, err := GetISCSIInfo(ctx, vid, req, targetInfo.Targets[0], lunInfo.AssignedNumber[0]) + if err != nil { + return "", status.Error(codes.Internal, err.Error()) + } + + utils.GetLogNODE(ctx, 5).Printf("attachBlockVolume: prepare mounting: %v", iscsiInfo) + fsType := req.GetVolumeCapability().GetMount().GetFsType() + mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags() + diskMounter := GetISCSIDiskMounter(iscsiInfo, false, fsType, mountOptions, "") + + utils.GetLogNODE(ctx, 5).Println("iSCSI Connector", "TargetPortals", diskMounter.connector.TargetPortals, + "Lun", diskMounter.connector.Lun, "TargetIqn", diskMounter.connector.TargetIqn, + "VolumeName", diskMounter.connector.VolumeName) + + util := &ISCSIUtil{} + + utils.GetLogNODE(ctx, 5).Println("attachBlockVolume: connecting disk", "diskMounter", diskMounter) + devicePath, err := util.ConnectDisk(ctx, *diskMounter) + if err != nil { + utils.GetLogNODE(ctx, 5).Println("attachBlockVolume: failed connecting the disk: %s", err.Error()) + return "", status.Error(codes.Internal, err.Error()) + } + + utils.GetLogNODE(ctx, 5).Println("attachBlockVolume: attached at: %s", devicePath) + return devicePath, nil +} + +func (zd *ZFSSADriver) nodeUnpublishBlockVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnpublishVolumeRequest, zvid *utils.VolumeId) (*csi.NodeUnpublishVolumeResponse, error) { + + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + diskUnmounter := GetISCSIDiskUnmounter(zvid) + + // Add decrement a node-local file reference count so we can keep track of when + // we can release the node's attach (RWO this reference count would only reach 1, + // RWM we may have many pods using the disk so we need to keep track) + + err := diskUnmounter.mounter.Unmount(targetPath) + if err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot unmount volume", + "volume_id", req.GetVolumeId(), "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + notMnt, mntErr := zd.NodeMounter.IsLikelyNotMountPoint(targetPath) + if mntErr != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot determine target path", + "target_path", targetPath, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + if notMnt { + if err := os.Remove(targetPath); err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot delete target path", + "target_path", targetPath, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + + return &csi.NodeUnpublishVolumeResponse{}, nil +} diff --git a/pkg/service/node_fs.go b/pkg/service/node_fs.go new file mode 100644 index 0000000..17adbf7 --- /dev/null +++ b/pkg/service/node_fs.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "fmt" + "os" + "strings" + + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (zd *ZFSSADriver) nodePublishFileSystem(ctx context.Context, token *zfssarest.Token, + req *csi.NodePublishVolumeRequest, vid *utils.VolumeId, mountOptions []string, + mode *csi.VolumeCapability_Mount) (*csi.NodePublishVolumeResponse, error) { + + targetPath := req.GetTargetPath() + notMnt, err := zd.NodeMounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } + + s := req.GetVolumeContext()["nfsServer"] + ep, found := req.GetVolumeContext()["mountpoint"] + if !found { + // The volume context of the volume provisioned from an existing share does not have the mountpoint. + // Use the share (corresponding to volumeAttributes.share of PV configuration) to get the mountpoint. + ep = req.GetVolumeContext()["share"] + } + + source := fmt.Sprintf("%s:%s", s, ep) + utils.GetLogNODE(ctx, 5).Println("nodePublishFileSystem", "mount_point", source) + + err = zd.NodeMounter.Mount(source, targetPath, "nfs", mountOptions) + if err != nil { + if os.IsPermission(err) { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + if strings.Contains(err.Error(), "invalid argument") { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (zd *ZFSSADriver) nodeUnpublishFilesystemVolume(token *zfssarest.Token, + ctx context.Context, req *csi.NodeUnpublishVolumeRequest, vid *utils.VolumeId) ( + *csi.NodeUnpublishVolumeResponse, error) { + + utils.GetLogNODE(ctx, 5).Println("nodeUnpublishFileSystem", "request", req) + + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + err := zd.NodeMounter.Unmount(targetPath) + if err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot unmount volume", + "volume_id", req.GetVolumeId(), "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + notMnt, mntErr := zd.NodeMounter.IsLikelyNotMountPoint(targetPath) + if mntErr != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot determine target path", + "target_path", targetPath, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + if notMnt { + if err := os.Remove(targetPath); err != nil { + utils.GetLogNODE(ctx, 2).Println("Cannot delete target path", + "target_path", targetPath, "error", err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + + return &csi.NodeUnpublishVolumeResponse{}, nil +} diff --git a/pkg/service/service.go b/pkg/service/service.go new file mode 100644 index 0000000..cabf42b --- /dev/null +++ b/pkg/service/service.go @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "errors" + "fmt" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc" + "gopkg.in/yaml.v2" + "io/ioutil" + "net" + "os" + "os/signal" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + // Default Log Level + DefaultLogLevel = "3" + DefaultCertPath = "/mnt/certs/zfssa.crt" + DefaultCredPath = "/mnt/zfssa/zfssa.yaml" +) + +type ZFSSADriver struct { + name string + nodeID string + version string + endpoint string + config config + NodeMounter Mounter + vCache volumeHashTable + sCache snapshotHashTable + ns *csi.NodeServer + cs *csi.ControllerServer + is *csi.IdentityServer +} + +type config struct { + Appliance string + User string + endpoint string + HostIp string + NodeName string + PodIp string + Secure bool + logLevel string + Certificate []byte + CertLocation string + CredLocation string +} + +// The structured data in the ZFSSA credentials file +type ZfssaCredentials struct { + Username string `yaml:username` + Password string `yaml:password` +} + +type accessType int + +// NonBlocking server + +type nonBlockingGRPCServer struct { + wg sync.WaitGroup + server *grpc.Server +} + +const ( + // Helpful size constants + Kib int64 = 1024 + Mib int64 = Kib * 1024 + Gib int64 = Mib * 1024 + Gib100 int64 = Gib * 100 + Tib int64 = Gib * 1024 + Tib100 int64 = Tib * 100 + + DefaultVolumeSizeBytes int64 = 50 * Gib + + mountAccess accessType = iota + blockAccess +) + +const ( + UsernamePattern string = `^[a-zA-Z][a-zA-Z0-9_\-\.]*$` + UsernameLength int = 255 +) + +type ZfssaBlockVolume struct { + VolName string `json:"volName"` + VolID string `json:"volID"` + VolSize int64 `json:"volSize"` + VolPath string `json:"volPath"` + VolAccessType accessType `json:"volAccessType"` +} + +// Creates and returns a new ZFSSA driver structure. +func NewZFSSADriver(driverName, version string) (*ZFSSADriver, error) { + + zd := new(ZFSSADriver) + + zd.name = driverName + zd.version = version + err := getConfig(zd) + if err != nil { + return nil, err + } + + zd.vCache.vHash = make(map[string]zVolumeInterface) + zd.sCache.sHash = make(map[string]*zSnapshot) + + utils.InitLogs(zd.config.logLevel, zd.name, version, zd.config.NodeName) + + err = zfssarest.InitREST(zd.config.Appliance, zd.config.Certificate, zd.config.Secure) + if err != nil { + return nil, err + } + + err = InitClusterInterface() + if err != nil { + return nil, err + } + + zd.is = newZFSSAIdentityServer(zd) + zd.cs = newZFSSAControllerServer(zd) + zd.ns = NewZFSSANodeServer(zd) + + return zd, nil +} + +// Gets the configuration and sanity checks it. Several environment variables values +// are retrieved: +// +// ZFSSA_TARGET The name or IP address of the appliance. +// NODE_NAME The name of the node on which the container is running. +// NODE_ID The ID of the node on which the container is running. +// CSI_ENDPOINT Unix socket the CSI driver will be listening on. +// ZFSSA_INSECURE Boolean specifying whether an appliance certificate is not required. +// ZFSSA_CERT Path to the certificate file (defaults to "/mnt/certs/zfssa.crt") +// ZFSSA_CRED Path to the credential file (defaults to "/mnt/zfssa/zfssa.yaml") +// HOST_IP IP address of the node. +// POD_IP IP address of the pod. +// LOG_LEVEL Log level to apply. +// +// Verifies the credentials are in the ZFSSA_CRED yaml file, does not verify their +// correctness. +func getConfig(zd *ZFSSADriver) error { + // Validate the ZFSSA credentials are available + credfile := strings.TrimSpace(getEnvFallback("ZFSSA_CRED", DefaultCredPath)) + if len(credfile) == 0 { + return errors.New(fmt.Sprintf("a ZFSSA credentials file location is required, current value: <%s>", + credfile)) + } + zd.config.CredLocation = credfile + _, err := os.Stat(credfile) + if os.IsNotExist(err) { + return errors.New(fmt.Sprintf("the ZFSSA credentials file is not present at location: <%s>", + credfile)) + } + + // Get the user from the credentials file, this can be stored in the config file without a problem + zd.config.User, err = zd.GetUsernameFromCred() + if err != nil { + return errors.New(fmt.Sprintf("Cannot get ZFSSA username: %s", err)) + } + + appliance := getEnvFallback("ZFSSA_TARGET", "") + zd.config.Appliance = strings.TrimSpace(appliance) + if zd.config.Appliance == "not-set" { + return errors.New("appliance name required") + } + + zd.config.NodeName = getEnvFallback("NODE_NAME", "") + if zd.config.NodeName == "" { + return errors.New("node name required") + } + + zd.config.endpoint = getEnvFallback("CSI_ENDPOINT", "") + if zd.config.endpoint == "" { + return errors.New("endpoint is required") + } else { + if !strings.HasPrefix(zd.config.endpoint, "unix://") { + return errors.New("endpoint is invalid") + } + s := strings.SplitN(zd.config.endpoint, "://", 2) + zd.config.endpoint = "/" + s[1] + err := os.RemoveAll(zd.config.endpoint) + if err != nil && !os.IsNotExist(err) { + return errors.New("failed to remove endpoint path") + } + } + + switch strings.ToLower(strings.TrimSpace(getEnvFallback("ZFSSA_INSECURE", "False"))) { + case "true": zd.config.Secure = false + case "false": zd.config.Secure = true + default: + return errors.New("ZFSSA_INSECURE value is invalid") + } + + if zd.config.Secure { + certfile := strings.TrimSpace(getEnvFallback("ZFSSA_CERT", DefaultCertPath)) + if len(certfile) == 0 { + return errors.New("a certificate is required") + } + _, err := os.Stat(certfile) + if os.IsNotExist(err) { + return errors.New("certificate does not exits") + } + zd.config.Certificate, err = ioutil.ReadFile(certfile) + if err != nil { + return errors.New("failed to read certificate") + } + } + + zd.config.HostIp = getEnvFallback("HOST_IP", "0.0.0.0") + zd.config.PodIp = getEnvFallback("POD_IP", "0.0.0.0") + zd.config.logLevel = getEnvFallback("LOG_LEVEL", DefaultLogLevel) + _, err = strconv.Atoi(zd.config.logLevel) + if err != nil { + return errors.New("invalid debug level") + } + return nil +} + +// Starts the CSI driver. This includes registering the different servers (Identity, Controller and Node) with +// the CSI framework and starting listening on the UNIX socket. + +var sigList = []os.Signal { + syscall.SIGTERM, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT, +} + +// Retrieves just the username from a credential file (zd.config.CredLocation) +func (zd *ZFSSADriver) GetUsernameFromCred() (string, error) { + yamlData, err := ioutil.ReadFile(zd.config.CredLocation) + if err != nil { + return "", errors.New(fmt.Sprintf("the ZFSSA credentials file <%s> could not be read: <%s>", + zd.config.CredLocation, err)) + } + + var yamlConfig ZfssaCredentials + err = yaml.Unmarshal(yamlData, &yamlConfig) + if err != nil { + return "", errors.New(fmt.Sprintf("the ZFSSA credentials file <%s> could not be parsed: <%s>", + zd.config.CredLocation, err)) + } + + if !isUsernameValid(yamlConfig.Username) { + return "", errors.New(fmt.Sprintf("ZFSSA username is invalid: <%s>", yamlConfig.Username)) + } + + return yamlConfig.Username, nil +} + +// Retrieves just the username from a credential file +func (zd *ZFSSADriver) GetPasswordFromCred() (string, error) { + yamlData, err := ioutil.ReadFile(zd.config.CredLocation) + if err != nil { + return "", errors.New(fmt.Sprintf("the ZFSSA credentials file <%s> could not be read: <%s>", + zd.config.CredLocation, err)) + } + + var yamlConfig ZfssaCredentials + err = yaml.Unmarshal(yamlData, &yamlConfig) + if err != nil { + return "", errors.New(fmt.Sprintf("the ZFSSA credentials file <%s> could not be parsed: <%s>", + zd.config.CredLocation, err)) + } + + return yamlConfig.Password, nil +} + +func (zd *ZFSSADriver) Run() { + // Refresh current information + _ = zd.updateVolumeList(nil) + _ = zd.updateSnapshotList(nil) + + // Create GRPC servers + s := new(nonBlockingGRPCServer) + + sigChannel := make(chan os.Signal, 1) + signal.Notify(sigChannel, sigList...) + + s.Start(zd.config.endpoint, *zd.is, *zd.cs, *zd.ns) + s.Wait(sigChannel) + s.Stop() + _ = os.RemoveAll(zd.config.endpoint) +} + +func (s *nonBlockingGRPCServer) Start(endpoint string, + ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + + s.wg.Add(1) + go s.serve(endpoint, ids, cs, ns) +} + +func (s *nonBlockingGRPCServer) Wait(ch chan os.Signal) { + for sig := range ch { + switch sig { + case syscall.SIGTERM, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT: + utils.GetLogCSID(nil, 5).Println("Termination signal received", "signal", sig) + return + default: + utils.GetLogCSID(nil, 5).Println("Signal received", "signal", sig) + continue + } + } +} + +func (s *nonBlockingGRPCServer) Stop() { + s.server.GracefulStop() + s.wg.Add(-1) +} + +func (s *nonBlockingGRPCServer) ForceStop() { + s.server.Stop() + s.wg.Add(-1) +} + +func (s *nonBlockingGRPCServer) serve(endpoint string, + ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + + listener, err := net.Listen("unix", endpoint) + if err != nil { + utils.GetLogCSID(nil, 2).Println("Failed to listen", "error", err) + return + } + + opts := []grpc.ServerOption{ grpc.UnaryInterceptor(interceptorGRPC) } + + server := grpc.NewServer(opts...) + s.server = server + + csi.RegisterIdentityServer(server, ids) + csi.RegisterControllerServer(server, cs) + csi.RegisterNodeServer(server, ns) + + utils.GetLogCSID(nil, 5).Println("Listening for connections", "address", endpoint) + + err = server.Serve(listener) + if err != nil { + utils.GetLogCSID(nil, 2).Println("Serve returned with error", "error", err) + } +} + +// Interceptor measuring the response time of the requests. +func interceptorGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + + // Get a new context with a list of loggers request specific. + newContext := utils.GetNewContext(ctx) + + // Calls the handler + utils.GetLogCSID(newContext, 4).Println("Request submitted", "method:", info.FullMethod) + start := time.Now() + rsp, err := handler(newContext, req) + utils.GetLogCSID(newContext, 4).Println("Request completed", "method:", info.FullMethod, + "duration:", time.Since(start), "error", err) + + return rsp, err +} + +// A local GetEnv utility function +func getEnvFallback(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +// validate username +func isUsernameValid(username string) bool { + if len(username) == 0 || len(username) > UsernameLength { + return false + } + + var validUsername = regexp.MustCompile(UsernamePattern).MatchString + return validUsername(username) +} diff --git a/pkg/service/snapshots.go b/pkg/service/snapshots.go new file mode 100644 index 0000000..7dc438d --- /dev/null +++ b/pkg/service/snapshots.go @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/ptypes/timestamp" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "net/http" + "sync/atomic" +) + +// Mount volume or BLock volume snapshot. +type zSnapshot struct { + bolt *utils.Bolt + refcount int32 + state volumeState + href string + zvol zVolumeInterface + id *utils.SnapshotId + numClones int + spaceUnique int64 + spaceData int64 + timeStamp *timestamp.Timestamp +} + +func newSnapshot(sid *utils.SnapshotId, zvol zVolumeInterface) *zSnapshot { + zsnap := new(zSnapshot) + zsnap.bolt = utils.NewBolt() + zsnap.zvol = zvol + zsnap.id = sid + zsnap.state = stateCreating + return zsnap +} + +func (zsnap *zSnapshot) create(ctx context.Context, token *zfssarest.Token) ( + *csi.CreateSnapshotResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("zsnap.create") + + snapinfo, httpStatus, err := zfssarest.CreateSnapshot(ctx, token, zsnap.zvol.getHref(), zsnap.id.Name) + if err != nil { + if httpStatus != http.StatusConflict { + zsnap.state = stateDeleted + return nil, err + } + // The creation failed because the source file system already has a snapshot + // with the same name. + if zsnap.getState() == stateCreated { + snapinfo, _, err := zfssarest.GetSnapshot(ctx, token, zsnap.zvol.getHref(), zsnap.id.Name) + if err != nil { + return nil, err + } + err = zsnap.setInfo(snapinfo) + if err != nil { + zsnap.state = stateDeleted + return nil, err + } + } + } else { + err = zsnap.setInfo(snapinfo) + if err != nil { + zsnap.state = stateDeleted + return nil, err + } + } + + return &csi.CreateSnapshotResponse { + Snapshot: &csi.Snapshot{ + SizeBytes: zsnap.spaceData, + SnapshotId: zsnap.id.String(), + SourceVolumeId: zsnap.id.VolumeId.String(), + CreationTime: zsnap.timeStamp, + ReadyToUse: true, + }}, nil +} + +func (zsnap *zSnapshot) delete(ctx context.Context, token *zfssarest.Token) ( + *csi.DeleteSnapshotResponse, error) { + + utils.GetLogCTRL(ctx, 5).Println("zsnap.delete") + + // Update the snapshot information. + httpStatus, err := zsnap.refreshDetails(ctx, token) + if err != nil { + if httpStatus != http.StatusNotFound { + return nil, err + } + zsnap.state = stateDeleted + return &csi.DeleteSnapshotResponse{}, nil + } + + if zsnap.getNumClones() > 0 { + return nil, status.Errorf(codes.FailedPrecondition, "Snapshot has (%d) dependents", zsnap.numClones) + } + + _, httpStatus, err = zfssarest.DeleteSnapshot(ctx, token, zsnap.href) + if err != nil && httpStatus != http.StatusNotFound { + return nil, err + } + + zsnap.state = stateDeleted + return &csi.DeleteSnapshotResponse{}, nil +} + + +func (zsnap *zSnapshot) getDetails(ctx context.Context, token *zfssarest.Token) (int, error) { + + utils.GetLogCTRL(ctx, 5).Println("zsnap.getDetails") + + snapinfo, httpStatus, err := zfssarest.GetSnapshot(ctx, token, zsnap.zvol.getHref(), zsnap.id.Name) + if err != nil { + return httpStatus, err + } + zsnap.timeStamp, err = utils.DateToUnix(snapinfo.Creation) + if err != nil { + return httpStatus, err + } + zsnap.numClones = snapinfo.NumClones + zsnap.spaceData = snapinfo.SpaceData + zsnap.spaceUnique = snapinfo.SpaceUnique + zsnap.href = snapinfo.Href + zsnap.state = stateCreated + return httpStatus, nil +} + +func (zsnap *zSnapshot) refreshDetails(ctx context.Context, token *zfssarest.Token) (int, error) { + snapinfo, httpStatus, err := zfssarest.GetSnapshot(ctx, token, zsnap.zvol.getHref(), zsnap.id.Name) + if err == nil { + zsnap.numClones = snapinfo.NumClones + zsnap.spaceData = snapinfo.SpaceData + zsnap.spaceUnique = snapinfo.SpaceUnique + } + return httpStatus, err +} + +// Populate the snapshot structure with the information provided +func (zsnap *zSnapshot) setInfo(snapinfo *zfssarest.Snapshot) error { + var err error + zsnap.timeStamp, err = utils.DateToUnix(snapinfo.Creation) + if err != nil { + return err + } + zsnap.numClones = snapinfo.NumClones + zsnap.spaceData = snapinfo.SpaceData + zsnap.spaceUnique = snapinfo.SpaceUnique + zsnap.href = snapinfo.Href + zsnap.state = stateCreated + return nil +} + +// Waits until the file system is available and, when it is, returns with its current state. +func (zsnap *zSnapshot) hold(ctx context.Context) volumeState { + atomic.AddInt32(&zsnap.refcount, 1) + return zsnap.state +} + +func (zsnap *zSnapshot) lock(ctx context.Context) volumeState { + zsnap.bolt.Lock(ctx) + return zsnap.state +} + +func (zsnap *zSnapshot) unlock(ctx context.Context) (int32, volumeState){ + zsnap.bolt.Unlock(ctx) + return zsnap.refcount, zsnap.state +} + +// Releases the file system and returns its current reference count. +func (zsnap *zSnapshot) release(ctx context.Context) (int32, volumeState) { + return atomic.AddInt32(&zsnap.refcount, -1), zsnap.state +} + +func (zsnap *zSnapshot) isBlock() bool { return zsnap.zvol.isBlock() } +func (zsnap *zSnapshot) getSourceVolume() zVolumeInterface { return zsnap.zvol } +func (zsnap *zSnapshot) getState() volumeState { return zsnap.state } +func (zsnap *zSnapshot) getName() string { return zsnap.id.Name } +func (zsnap *zSnapshot) getStringId() string { return zsnap.id.String() } +func (zsnap *zSnapshot) getStringSourceId() string { return zsnap.id.VolumeId.String() } +func (zsnap *zSnapshot) getHref() string { return zsnap.href } +func (zsnap *zSnapshot) getSize() int64 { return zsnap.spaceData } +func (zsnap *zSnapshot) getCreationTime() *timestamp.Timestamp { return zsnap.timeStamp } +func (zsnap *zSnapshot) getNumClones() int { return zsnap.numClones } + +func zfssaSnapshotList2csiSnapshotList(ctx context.Context, zfssa string, + snapList []zfssarest.Snapshot) []*csi.ListSnapshotsResponse_Entry { + + utils.GetLogCTRL(ctx, 5).Println("zfssaSnapshotList2csiSnapshotList") + + entries := make([]*csi.ListSnapshotsResponse_Entry, 0, len(snapList)) + + for _, snapInfo := range snapList { + sid, err := utils.SnapshotIdStringFromHref(zfssa, snapInfo.Href) + if err != nil { + continue + } + vid, err := utils.VolumeIdStringFromHref(zfssa, snapInfo.Href) + if err != nil { + continue + } + creationTime, err := utils.DateToUnix(snapInfo.Creation) + if err != nil { + continue + } + entry := new(csi.ListSnapshotsResponse_Entry) + entry.Snapshot = &csi.Snapshot{ + SnapshotId: sid, + SizeBytes: snapInfo.SpaceData, + SourceVolumeId: vid, + CreationTime: creationTime, + ReadyToUse: true, + } + entries = append(entries, entry) + } + return entries +} diff --git a/pkg/service/volumes.go b/pkg/service/volumes.go new file mode 100644 index 0000000..f68525e --- /dev/null +++ b/pkg/service/volumes.go @@ -0,0 +1,647 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package service + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "github.com/oracle/zfssa-csi-driver/pkg/zfssarest" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "net/http" + "sync" +) + +// This file contains the definition of the volume interface. A volume can be +// a block device or a file system mounted over a network protocol. Both types +// of volumes must satisfy the interface defined here (zVolumeInterface). +// +// Volume Access Control +// --------------------- +// Under the subtitle "Concurrency", the CSI specification states: +// +// "In general the Cluster Orchestrator (CO) is responsible for ensuring +// that there is no more than one call “in-flight” per volume at a given +// time. However, in some circumstances, the CO MAY lose state (for +// example when the CO crashes and restarts), and MAY issue multiple calls +// simultaneously for the same volume. The plugin SHOULD handle this as +// gracefully as possible." +// +// In order to handle this situation, methods of the ZFSSA driver defined in this +// file guarantee that only one request (or Go routine) has access to a volume at +// any time. Those methods are: +// +// newVolume() +// lookupVolume() +// releaseVolume() +// +// The first 2 methods, if successful, return a volume and exclusive access to it +// to the caller. When the volume is not needed anymore, exclusive access must +// be relinquished by calling releaseVolume(). +// +// Snapshot Access Control +// ----------------------- +// The same semantics as volumes apply to snapshots. The snapshot methods are: +// +// newSnapshot() +// lookupSnapshot() +// releaseSnapshot() +// +// As for volumes, the first 2 methods, if successful, return a snapshot and +// exclusive access to it to the caller. It also gives exclusive access to the +// volume source of the snapshot. When the snapshot is not needed any more, +// calling releaseSnapshot() relinquishes exclusive access to the snapshot and +// volume source. +// + +type volumeState int + +const ( + stateCreating volumeState = iota + stateCreated + stateDeleted +) + +// Interface that all ZFSSA type volumes (mount and block) must satisfy. +type zVolumeInterface interface { + create(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) + delete(ctx context.Context, token *zfssarest.Token, + ) (*csi.DeleteVolumeResponse, error) + controllerPublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerPublishVolumeRequest, nodeName string) (*csi.ControllerPublishVolumeResponse, error) + controllerUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) + validateVolumeCapabilities(ctx context.Context, token *zfssarest.Token, + req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) + controllerExpandVolume(ctx context.Context, token *zfssarest.Token, + req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) + nodeStageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) + nodeUnstageVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) + nodePublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) + nodeUnpublishVolume(ctx context.Context, token *zfssarest.Token, + req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) + nodeGetVolumeStats(ctx context.Context, token *zfssarest.Token, + req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) + cloneSnapshot(ctx context.Context, token *zfssarest.Token, + req *csi.CreateVolumeRequest, zsnap *zSnapshot) (*csi.CreateVolumeResponse, error) + getDetails(ctx context.Context, token *zfssarest.Token) (int, error) + setInfo(volInfo interface{}) + getSnapshotsList(context.Context, *zfssarest.Token) ([]*csi.ListSnapshotsResponse_Entry, error) + hold(ctx context.Context) volumeState + release(ctx context.Context) (int32, volumeState) + lock(ctx context.Context) volumeState + unlock(ctx context.Context) (int32, volumeState) + getState() volumeState + getName() string + getHref() string + getVolumeID() *utils.VolumeId + getCapacity() int64 + isBlock() bool +} + + +// This method must be called when the possibility of the volume not existing yet exists. +// The following 3 scenarios are possible: +// +// * The volume doesn't exists yet or it exists in the appliance but is not in the +// volume cache yet. Either way, a structure representing the volume is created +// in the stateCreating state, stored in the cache and a reference returned to the +// caller. +// * A structure representing the volume already exists in the cache and is in the +// stateCreated state. A reference is returned to the caller. +// * A structure representing the volume already exists in the cache and is NOT in +// the stateCreated state. This means the CO probably lost state and submitted multiple +// simultaneous requests for this volume. In this case an error is returned. +// +func (zd *ZFSSADriver) newVolume(ctx context.Context, pool, project, name string, + block bool) (zVolumeInterface, error) { + + var vid *utils.VolumeId + var zvolNew zVolumeInterface + if block { + vid = utils.NewVolumeId(utils.BlockVolume, zd.config.Appliance, pool, project, name) + zvolNew = newLUN(vid) + } else { + vid = utils.NewVolumeId(utils.MountVolume, zd.config.Appliance, pool, project, name) + zvolNew = newFilesystem(vid) + } + + zd.vCache.Lock(ctx) + zvol := zd.vCache.lookup(ctx, name) + if zvol != nil { + // Volume already known. + utils.GetLogCTRL(ctx, 5).Println("zd.newVolume", "request", ) + zvol.hold(ctx) + zd.vCache.Unlock(ctx) + if zvol.lock(ctx) != stateCreated { + zd.releaseVolume(ctx, zvol) + return nil, status.Errorf(codes.Aborted, "volume busy (%s)", vid.String()) + } + return zvol, nil + } + + zd.vCache.add(ctx, name, zvolNew) + zvolNew.hold(ctx) + zvolNew.lock(ctx) + zd.vCache.Unlock(ctx) + return zvolNew, nil +} + +// This method must be called in every CSI method receiving a request with a volume ID. +// The list of known volumes is scanned. If no volume with a matching ID is found, the +// appliance is queried. If the appliance knows of the volume, the local list is updated +// and the volume is returned to the caller. When the volume returned is not needed +// anymore, the method releaseVolume() must be called. +func (zd *ZFSSADriver) lookupVolume(ctx context.Context, token *zfssarest.Token, + volumeId string) (zVolumeInterface, error) { + + vid, err := utils.VolumeIdFromString(volumeId) + if err != nil { + return nil, err + } + + // Check first in the list of volumes if the volume is already known. + zd.vCache.RLock(ctx) + + zvol := zd.vCache.lookup(ctx, vid.Name) + if zvol != nil { + zvol.hold(ctx) + zd.vCache.RUnlock(ctx) + if zvol.lock(ctx) != stateCreated { + zd.releaseVolume(ctx, zvol) + return nil, status.Errorf(codes.Aborted, "volume busy (%s)", volumeId) + } + return zvol, nil + } + + zd.vCache.RUnlock(ctx) + + // Create a context for the new volume. The new context will act as a place holder + // for the name passed in. + zvol, err = zd.newVolume(ctx, vid.Pool, vid.Project, vid.Name, vid.Type == utils.BlockVolume) + if err != nil { + return nil, err + } + + switch zvol.getState() { + case stateCreating: // We check with the appliance. + httpStatus, err := zvol.getDetails(ctx, token) + if err != nil { + zd.releaseVolume(ctx, zvol) + if httpStatus == http.StatusNotFound { + return nil, status.Error(codes.NotFound, "Volume (%s) not found") + } + return nil, err + } + return zvol, nil + case stateCreated: // Another Goroutine beat us to it. + return zvol, nil + default: + zd.releaseVolume(ctx, zvol) + return nil, status.Error(codes.NotFound, "Volume (%s) not found") + } +} + +// Releases the volume reference and exclusive access to the volume. +func (zd *ZFSSADriver) releaseVolume(ctx context.Context, zvol zVolumeInterface) { + zd.vCache.RLock(ctx) + refCount, state := zvol.release(ctx) + if refCount == 0 && state != stateCreated { + zd.vCache.RUnlock(ctx) + zd.vCache.Lock(ctx) + refCount, state = zvol.unlock(ctx) + if refCount == 0 && state != stateCreated { + zd.vCache.delete(ctx, zvol.getName()) + } + zd.vCache.Unlock(ctx) + } else { + zvol.unlock(ctx) + zd.vCache.RUnlock(ctx) + } + utils.GetLogCTRL(ctx, 5).Printf(" zd.releaseVolume is done") +} + +// If a snapshot with the passed in name already exists, it is returned. If it doesn't exist, +// a new snapshot structure is created and returned. This method could fail or reasons: +// +// 1) A snapshot with the passed in name already exists but the volume source +// is not the volume source passed in. +// 2) A snapshot with the passed in name already exists but is not in the stateCreated +// state (or stable state). As for volumes, This would mean the CO lost state and +// issued simultaneous requests for the same snapshot. +// +// If the call is successful, that caller has exclusive access to the snapshot and its volume +// source. When the snapshot returned is not needed anymore, the method releaseSnapshot() +// must be called. +func (zd *ZFSSADriver) newSnapshot(ctx context.Context, token *zfssarest.Token, + name, sourceId string) (*zSnapshot, error) { + + zvol, err := zd.lookupVolume(ctx, token, sourceId) + if err != nil { + return nil, err + } + + sid := utils.NewSnapshotId(zvol.getVolumeID(), name) + + zd.sCache.Lock(ctx) + + zsnap := zd.sCache.lookup(ctx, name) + if zsnap == nil { + // Snapshot doesn't exist or is unknown. + zsnap := newSnapshot(sid, zvol) + _ = zsnap.hold(ctx) + zd.sCache.add(ctx, name, zsnap) + zd.sCache.Unlock(ctx) + zsnap.lock(ctx) + return zsnap, nil + } + + // We don't have exclusive access to the snapshot yet but it's safe to access + // here the field id of the snapshot. + if zsnap.getStringSourceId() != sourceId { + zd.sCache.Unlock(ctx) + zd.releaseVolume(ctx, zvol) + return nil, status.Errorf(codes.AlreadyExists, + "snapshot (%s) already exists with different source", name) + } + + zsnap.hold(ctx) + zd.sCache.Unlock(ctx) + if zsnap.lock(ctx) != stateCreated { + zd.releaseSnapshot(ctx, zsnap) + return nil, status.Errorf(codes.Aborted, "snapshot (%s) is busy", name) + } + + return zsnap, nil +} + +// Looks up a snapshot using a volume ID (in string form). If successful, the caller gets +// exclusive access to the returned snapshot and its volume source. This method could fail +// for the following reasons: +// +// 1) The source volume cannot be found locally or in the appliance. +// 2) The snapshot exists but is in an unstable state. This would mean the +// CO lost state and issued multiple simultaneous requests for the same +// snapshot. +// 3) There's an inconsistency between what the appliance thinks the volume +// source is and what the existing snapshot says it is (a panic is issued). +// 4) The snapshot cannot be found locally or in the appliance. +func (zd *ZFSSADriver) lookupSnapshot(ctx context.Context, token *zfssarest.Token, + snapshotId string) (*zSnapshot, error) { + + var zsnap *zSnapshot + var zvol zVolumeInterface + var err error + + // Break up the string into its components + sid, err := utils.SnapshotIdFromString(snapshotId) + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("Snapshot ID was invalid", "snapshot_id", snapshotId) + return nil, status.Errorf(codes.NotFound, "Unknown snapshot (%s)", snapshotId) + } + + // Get first exclusive access to the volume source. + zvol, err = zd.lookupVolume(ctx, token, sid.GetVolumeId().String()) + if err != nil { + return nil, err + } + + zd.sCache.RLock(ctx) + zsnap = zd.sCache.lookup(ctx, sid.Name) + if zsnap != nil { + if zsnap.getSourceVolume() != zvol { + // This is a serious problem. It means the volume source found using + // the snapshot id is not the same as the volume source already is the + // snapshot structure. + panic("snapshot id and volume source inconsistent") + } + zsnap.hold(ctx) + zd.sCache.RUnlock(ctx) + if zsnap.lock(ctx) != stateCreated { + zd.releaseSnapshot(ctx, zsnap) + err = status.Errorf(codes.Aborted, "snapshot (%s) is busy", snapshotId) + zsnap = nil + } else { + err = nil + } + return zsnap, err + } + zd.sCache.RUnlock(ctx) + zd.releaseVolume(ctx, zvol) + + // Query the appliance. + zsnap, err = zd.newSnapshot(ctx, token, sid.Name, sid.VolumeId.String()) + if err != nil { + return nil, err + } + + switch zsnap.getState() { + case stateCreating: // We check with the appliance. + _, err = zsnap.getDetails(ctx, token) + if err != nil { + zd.releaseSnapshot(ctx, zsnap) + return nil, err + } + return zsnap, nil + case stateCreated: // Another Goroutine beat us to it. + return zsnap, nil + default: + zd.releaseSnapshot(ctx, zsnap) + return nil, err + } +} + +// Releases the snapshot reference and exclusive access to the snapshot. Holding +// a snapshot reference and having exclusive access to it means the caller also have +// a reference to the source volume and exclusive access to it. The volume source +// is also released. +func (zd *ZFSSADriver) releaseSnapshot(ctx context.Context, zsnap *zSnapshot) { + zvol := zsnap.getSourceVolume() + zd.sCache.RLock(ctx) + refCount, state := zsnap.release(ctx) + if refCount == 0 && state != stateCreated { + zd.sCache.RUnlock(ctx) + zd.sCache.Lock(ctx) + refCount, state = zsnap.unlock(ctx) + if refCount == 0 && state != stateCreated { + zd.sCache.delete(ctx, zsnap.getName()) + } + zd.sCache.Unlock(ctx) + } else { + zd.sCache.RUnlock(ctx) + zsnap.unlock(ctx) + } + zd.releaseVolume(ctx, zvol) +} + +// Asks the appliance for the list of LUNs and filesystems, updates the local list of +// volumes and returns a list in CSI format. +func (zd *ZFSSADriver) getVolumesList(ctx context.Context) ([]*csi.ListVolumesResponse_Entry, error) { + + err := zd.updateVolumeList(ctx) + if err != nil { + return nil, err + } + + zd.vCache.RLock(ctx) + entries := make([]*csi.ListVolumesResponse_Entry, len(zd.sCache.sHash)) + for _, zvol := range zd.vCache.vHash { + entry := new(csi.ListVolumesResponse_Entry) + entry.Volume = &csi.Volume{ + VolumeId: zvol.getVolumeID().String(), + CapacityBytes: zvol.getCapacity(), + } + entries = append(entries, entry) + } + zd.vCache.RUnlock(ctx) + + return entries, nil +} + +// Retrieves the list of LUNs and filesystems from the appliance and updates +// the local list. +func (zd *ZFSSADriver) updateVolumeList(ctx context.Context) error { + + fsChan := make(chan error) + lunChan := make(chan error) + go zd.updateFilesystemList(ctx, fsChan) + go zd.updateLunList(ctx, lunChan) + errfs := <- fsChan + errlun := <- lunChan + + if errfs != nil { + return errfs + } else if errlun != nil { + return errlun + } + + return nil +} + +// Asks the appliance for the list of filesystems and updates the local list of volumes. +func (zd *ZFSSADriver) updateFilesystemList(ctx context.Context, out chan<- error) { + + utils.GetLogCTRL(ctx, 5).Println("zd.updateFilesystemList") + + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + out <- err + } + token := zfssarest.LookUpToken(user, password) + fsList, err := zfssarest.GetFilesystems(ctx, token, "", "") + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("zd.updateFilesystemList failed", "error", err.Error()) + } else { + for _, fsInfo := range fsList { + zvol, err := zd.newVolume(ctx, fsInfo.Pool, fsInfo.Project, fsInfo.Name, false) + if err != nil { + continue + } + zvol.setInfo(&fsInfo) + zd.releaseVolume(ctx, zvol) + } + } + out <- err +} + +// Asks the appliance for the list of LUNs and updates the local list of volumes. +func (zd *ZFSSADriver) updateLunList(ctx context.Context, out chan<- error) { + + utils.GetLogCTRL(ctx, 5).Println("zd.updateLunList") + + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + out <- err + } + token := zfssarest.LookUpToken(user, password) + + lunList, err := zfssarest.GetLuns(ctx, token, "", "") + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("zd.updateLunList failed", "error", err.Error()) + } else { + for _, lunInfo := range lunList { + zvol, err := zd.newVolume(ctx, lunInfo.Pool, lunInfo.Project, lunInfo.Name, true) + if err != nil { + continue + } + zvol.setInfo(&lunInfo) + zd.releaseVolume(ctx, zvol) + } + } + out <- err +} + +// Asks the appliance for the list of its snapshots and returns it in CSI format. The +// local list of snapshots is updated in the process. +func (zd *ZFSSADriver) getSnapshotList(ctx context.Context) ([]*csi.ListSnapshotsResponse_Entry, error) { + + utils.GetLogCTRL(ctx, 5).Println("zd.getSnapshotList") + + err := zd.updateSnapshotList(ctx) + if err != nil { + return nil, err + } + + zd.sCache.RLock(ctx) + entries := make([]*csi.ListSnapshotsResponse_Entry, 0, len(zd.sCache.sHash)) + for _, zsnap := range zd.sCache.sHash { + entry := new(csi.ListSnapshotsResponse_Entry) + entry.Snapshot = &csi.Snapshot { + SizeBytes: zsnap.getSize(), + SnapshotId: zsnap.getStringId(), + SourceVolumeId: zsnap.getStringSourceId(), + CreationTime: zsnap.getCreationTime(), + ReadyToUse: zsnap.getState() == stateCreated, + } + entries = append(entries, entry) + } + zd.sCache.RUnlock(ctx) + + return entries, nil +} + +// Requests the list of snapshots from the appliance and updates the local list. Only +// snapshots that can be identified as filesytem snapshots or lun snapshots are kept. +func (zd *ZFSSADriver) updateSnapshotList(ctx context.Context) error { + + utils.GetLogCTRL(ctx, 5).Println("zd.updateSnapshotList") + + user, password, err := zd.getUserLogin(ctx, nil) + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("Authentication error", "error", err.Error()) + return err + } + + token := zfssarest.LookUpToken(user, password) + snapList, err := zfssarest.GetSnapshots(ctx, token, "") + if err != nil { + utils.GetLogCTRL(ctx, 2).Println("zd.updateSnapshotList failed", "error", err.Error()) + return err + } + + for _, snapInfo := range snapList { + sid, err := utils.SnapshotIdFromHref(token.Name, snapInfo.Href) + if err != nil { + continue + } + zsnap, err := zd.newSnapshot(ctx, token, snapInfo.Name, sid.VolumeId.String()) + if err != nil { + continue + } + err = zsnap.setInfo(&snapInfo) + if err != nil { + continue + } + zd.releaseSnapshot(ctx, zsnap) + } + + return nil +} + +func compareCapacityRange(req *csi.CapacityRange, capacity int64) bool { + if req != nil { + if req.LimitBytes != 0 && req.LimitBytes < capacity { + return false + } + if req.RequiredBytes != 0 && req.RequiredBytes > capacity { + return false + } + } + return true +} + +func compareCapabilities(req []*csi.VolumeCapability, cur []csi.VolumeCapability_AccessMode, block bool) bool { + + hasSupport := func(cap *csi.VolumeCapability, caps []csi.VolumeCapability_AccessMode, block bool) bool { + for _, c := range caps { + if ((block && cap.GetBlock() != nil) || (!block && cap.GetBlock() == nil)) && + cap.GetAccessMode().Mode == c.Mode { + return true + } + } + return false + } + + foundAll := true + for _, c := range req { + if !hasSupport(c, cur, block) { + foundAll = false + break + } + } + return foundAll +} + +type volumeHashTable struct { + vMutex sync.RWMutex + vHash map[string]zVolumeInterface +} + +func (h *volumeHashTable) add(ctx context.Context, key string, zvol zVolumeInterface) { + h.vHash[key] = zvol +} + +func (h *volumeHashTable) delete(ctx context.Context, key string) { + delete(h.vHash, key) +} + +func (h *volumeHashTable) lookup(ctx context.Context, key string) zVolumeInterface { + return h.vHash[key] +} + +func (h *volumeHashTable) Lock(ctx context.Context) { + h.vMutex.Lock() +} + +func (h *volumeHashTable) Unlock(ctx context.Context) { + h.vMutex.Unlock() +} + +func (h *volumeHashTable) RLock(ctx context.Context) { + h.vMutex.RLock() +} + +func (h *volumeHashTable) RUnlock(ctx context.Context) { + h.vMutex.RUnlock() +} + +type snapshotHashTable struct { + sMutex sync.RWMutex + sHash map[string]*zSnapshot +} + +func (h *snapshotHashTable) add(ctx context.Context, key string, zsnap *zSnapshot) { + h.sHash[key] = zsnap +} + +func (h *snapshotHashTable) delete(ctx context.Context, key string) { + delete(h.sHash, key) +} + +func (h *snapshotHashTable) lookup(ctx context.Context, key string) *zSnapshot { + return h.sHash[key] +} + +func (h *snapshotHashTable) Lock(ctx context.Context) { + h.sMutex.Lock() +} + +func (h *snapshotHashTable) Unlock(ctx context.Context) { + h.sMutex.Unlock() +} + +func (h *snapshotHashTable) RLock(ctx context.Context) { + h.sMutex.RLock() +} + +func (h *snapshotHashTable) RUnlock(ctx context.Context) { + h.sMutex.RUnlock() +} diff --git a/pkg/utils/log_utils.go b/pkg/utils/log_utils.go new file mode 100644 index 0000000..5bbca1e --- /dev/null +++ b/pkg/utils/log_utils.go @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package utils + +import ( + "flag" + "fmt" + "golang.org/x/net/context" + "io/ioutil" + "k8s.io/klog" + "log" + "os" + "strconv" + "sync/atomic" +) + +const ( + CSID int = iota // CSI Driver + CTRL // Controller Service + NODE // Node Service + IDTY // Identity Service + REST // REST interface + UTIL // Utilities that may be controller or node + SENTINEL +) + +const MAX_LEVEL int = 5 + +type zLogger struct { + prefix string + logger [MAX_LEVEL]*log.Logger +} + +// Type of the key being used to add the array of loggers to the context. +type zLoggersKey string + +var ( + reqCounter uint64 + logLevelStr string + logLevel int + loggersPrefix [SENTINEL]string + loggersTable [MAX_LEVEL]*log.Logger + loggersKey zLoggersKey = "zloggers" + loggerNOP *log.Logger +) + +// Log service initialization. The original loggers are created with a prefix identifying the +// service (service, controller, node, indentifier, REST interface and utility), the node, +// the driver and its version. These loggers will be clone for each request. +func InitLogs(level, driverName, version, nodeID string) { + + klog.InitFlags(nil) + + logLevelStr = level + logLevel, _ = strconv.Atoi(level) + reqCounter = 0 + + _ = flag.Set("logtostderr", "true") + _ = flag.Set("v", logLevelStr) + + loggersPrefix[CSID] = nodeID + "/" + driverName + "/" + version + "\n\t" + loggersPrefix[CTRL] = nodeID + "/" + driverName + "/" + version + "\n\t" + loggersPrefix[NODE] = nodeID + "/" + driverName + "/" + version + "\n\t" + loggersPrefix[IDTY] = nodeID + "/" + driverName + "/" + version + "\n\t" + loggersPrefix[REST] = nodeID + "/" + driverName + "/" + version + "\n\t" + loggersPrefix[UTIL] = nodeID + "/" + driverName + "/" + version + "\n\t" + + for i := 0; i < MAX_LEVEL; i++ { + loggersTable[i] = log.New(os.Stdout, loggersPrefix[CSID] + "*** ", + log.Lmsgprefix|log.Ldate|log.Ltime|log.Lshortfile) + } + + loggerNOP = log.New(ioutil.Discard, "NOP logger", log.LstdFlags) +} + +// Creates a new context by duplicating the context passed in and adding a array of loggers +// with a value added. The value is unique and is generated in this function. That value will +// be systematically displayed each time any of the loggers in the array are called. +func GetNewContext(ctx context.Context) context.Context { + + loggers := new([SENTINEL]zLogger) + reqNum := fmt.Sprintf("%d", atomic.AddUint64(&reqCounter, 1)) + + for i := 0; i < SENTINEL; i++ { + loggers[i].prefix = loggersPrefix[i] + reqNum + " " + for j := 0; j < MAX_LEVEL; j++ { + loggers[i].logger[j] = nil + } + } + + return context.WithValue(ctx, loggersKey, loggers) +} + +// Return the appropriate logger based on the service and level provided. +func getLogger(ctx context.Context, sel int, level int) *log.Logger { + + if level > logLevel { + return loggerNOP + } + + level-- + + if ctx != nil { + loggers := ctx.Value(loggersKey).(*[SENTINEL]zLogger) + if loggers == nil { + panic("context without loggers") + } + if loggers[sel].logger[level] == nil { + // No logger has yet been created for this level + loggers[sel].logger[level] = log.New(os.Stdout, loggers[sel].prefix, + log.Lmsgprefix|log.Ldate|log.Ltime|log.Lshortfile) + } + return loggers[sel].logger[level] + } + + return loggersTable[level] +} + +// Public function returning the appropriate logger. +func GetLogCSID(ctx context.Context, level int) *log.Logger { return getLogger(ctx, CSID, level) } +func GetLogCTRL(ctx context.Context, level int) *log.Logger { return getLogger(ctx, CTRL, level) } +func GetLogNODE(ctx context.Context, level int) *log.Logger { return getLogger(ctx, NODE, level) } +func GetLogIDTY(ctx context.Context, level int) *log.Logger { return getLogger(ctx, IDTY, level) } +func GetLogREST(ctx context.Context, level int) *log.Logger { return getLogger(ctx, REST, level) } +func GetLogUTIL(ctx context.Context, level int) *log.Logger { return getLogger(ctx, UTIL, level) } diff --git a/pkg/utils/sync.go b/pkg/utils/sync.go new file mode 100644 index 0000000..1a954e1 --- /dev/null +++ b/pkg/utils/sync.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package utils + +import ( + "context" + "sync" +) + +type Bolt struct { + mutex sync.Mutex + cv *sync.Cond + context context.Context +} + +func NewBolt() *Bolt { + bolt := new(Bolt) + bolt.cv = sync.NewCond(&bolt.mutex) + return bolt +} + +func (l *Bolt) Lock(ctx context.Context) { + l.mutex.Lock() + for l.context != nil { + l.cv.Wait() + } + l.context = ctx + l.mutex.Unlock() +} + +func (l *Bolt) Unlock(ctx context.Context) { + l.mutex.Lock() + if l.context != ctx { + panic("wrong owner unlocking fs") + } + l.context = nil + l.cv.Signal() + l.mutex.Unlock() +} diff --git a/pkg/utils/volume_id.go b/pkg/utils/volume_id.go new file mode 100644 index 0000000..95bf48a --- /dev/null +++ b/pkg/utils/volume_id.go @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package utils + +import ( + "fmt" + "github.com/golang/protobuf/ptypes/timestamp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + VolumeIdLen = 6 + VolumeHandleLen = 8 + SnapshotIdLen = 7 + VolumeHrefLen = 10 + SnapshotHrefLen = 12 +) + +const ( + BlockVolume string = "lun" + MountVolume = "mnt" +) + +const ( + ResourceNamePattern string = `^[a-zA-Z0-9_\-\.\:]+$` + ResourceNameLength int = 64 +) + +// Volume ID +// --------- +// This structure is what identifies a volume (lun or filesystem). +type VolumeId struct { + Type string + Zfssa string + Pool string + Project string + Name string +} + +func NewVolumeId(vType, zfssaName, pool, project, name string) *VolumeId { + return &VolumeId{ + Type: vType, + Zfssa: zfssaName, + Pool: pool, + Project: project, + Name: name, + } +} + +func VolumeIdStringFromHref(zfssa, hRef string) (string, error) { + result := strings.Split(hRef, "/") + if len(result) < VolumeHrefLen { + return "", status.Errorf(codes.NotFound, + "Volume ID (%s) contains insufficient components (%d)",hRef, VolumeHrefLen) + } + + var vType string + switch result[8] { + case "filesystems": + vType = MountVolume + case "luns": + vType = BlockVolume + default: + return "", status.Errorf(codes.NotFound,"Invalid snapshot href (%s)", hRef) + } + + return fmt.Sprintf("/%v/%v/%v/%v/%v", + vType, + zfssa, + result[5], + result[7], + result[9]), nil +} + +func VolumeIdFromString(volumeId string) (*VolumeId, error) { + result := strings.Split(volumeId, "/") + + var pool, project, share string + switch result[1] { + case "nfs", "iscsi": + if len(result) < VolumeHandleLen { + return nil, status.Errorf(codes.NotFound, + "Volume ID/Handle (%s) contains insufficient components (%d)", volumeId, VolumeHandleLen) + } + pool = result[4] + project = result[6] + share = result[7] + default: + if len(result) < VolumeIdLen { + return nil, status.Errorf(codes.NotFound, + "Volume ID (%s) contains insufficient components (%d)", volumeId, VolumeIdLen) + } + pool = result[3] + project = result[4] + share = result[5] + } + + if !IsResourceNameValid(pool) { + return nil, status.Errorf(codes.InvalidArgument, "pool name is invalid (%s)", pool) + } + + if !IsResourceNameValid(project) { + return nil, status.Errorf(codes.InvalidArgument, "project name is invalid (%s)", project) + } + + if !IsResourceNameValid(share) { + return nil, status.Errorf(codes.InvalidArgument, "share name is invalid (%s)", share) + } + + return &VolumeId{ + Type: result[1], + Zfssa: result[2], + Pool: pool, + Project: project, + Name: share, + }, nil +} + +func (zvi *VolumeId) String() string { + return fmt.Sprintf("/%v/%v/%v/%v/%v", zvi.Type, zvi.Zfssa, zvi.Pool, zvi.Project, zvi.Name) +} + +func (zvi *VolumeId) IsBlock() bool { + switch zvi.Type { + case BlockVolume: return true + case MountVolume: return false + } + return false +} + +// Snapshot ID +// ----------- +// This structure is what identifies a volume (lun or filesystem). +type SnapshotId struct { + VolumeId *VolumeId + Name string +} + +func NewSnapshotId(volumeId *VolumeId, snapshotName string) *SnapshotId { + return &SnapshotId{ + VolumeId: volumeId, + Name: snapshotName, + } +} + +func SnapshotIdFromString(snapshotId string) (*SnapshotId, error) { + + result := strings.Split(snapshotId, "/") + if len(result) < SnapshotIdLen { + return nil, status.Errorf(codes.NotFound, + "Volume ID (%s) contains insufficient components (%d)", snapshotId, SnapshotIdLen) + } + + return &SnapshotId{ + VolumeId: &VolumeId{ + Type: result[1], + Zfssa: result[2], + Pool: result[3], + Project: result[4], + Name: result[5] }, + Name: result[6] }, nil +} + +func SnapshotIdFromHref(zfssa, hRef string) (*SnapshotId, error) { + result := strings.Split(hRef, "/") + if len(result) < SnapshotHrefLen { + return nil, status.Errorf(codes.NotFound, + "Snapshot ID (%s) contains insufficient components (%d)",hRef, SnapshotHrefLen) + } + + if result[10] != "snapshots" { + return nil, status.Errorf(codes.NotFound,"Invalid snapshot href (%s)", hRef) + } + + var vType string + switch result[8] { + case "filesystems": + vType = MountVolume + case "luns": + vType = BlockVolume + default: + return nil, status.Errorf(codes.NotFound,"Invalid snapshot href (%s)", hRef) + } + + return &SnapshotId{ + VolumeId: &VolumeId{ + Type: vType, + Zfssa: zfssa, + Pool: result[5], + Project: result[7], + Name: result[9] }, + Name: result[11] }, nil +} + +func SnapshotIdStringFromHref(zfssa, hRef string) (string, error) { + result := strings.Split(hRef, "/") + if len(result) < SnapshotHrefLen { + return "", status.Errorf(codes.NotFound, + "Snapshot ID (%s) contains insufficient components (%d)",hRef, SnapshotHrefLen) + } + + if result[10] != "snapshots" { + return "", status.Errorf(codes.NotFound,"Invalid snapshot href (%s)", hRef) + } + + var vType string + switch result[8] { + case "filesystems": + vType = MountVolume + case "luns": + vType = BlockVolume + default: + return "", status.Errorf(codes.NotFound,"Invalid snapshot href (%s)", hRef) + } + + return fmt.Sprintf("/%v/%v/%v/%v/%v/%v", + vType, + zfssa, + result[5], + result[7], + result[9], + result[11]), nil +} + +func (zsi *SnapshotId) String() string { + return fmt.Sprintf("/%v/%v/%v/%v/%v/%v", + zsi.VolumeId.Type, + zsi.VolumeId.Zfssa, + zsi.VolumeId.Pool, + zsi.VolumeId.Project, + zsi.VolumeId.Name, + zsi.Name) +} + +func (zsi *SnapshotId) GetVolumeId() *VolumeId { + return zsi.VolumeId +} + +func DateToUnix(date string) (*timestamp.Timestamp, error) { + year, err := strconv.Atoi(date[0:4]) + if err == nil { + month, err := strconv.Atoi(date[5:7]) + if err == nil { + day, err := strconv.Atoi(date[8:10]) + if err == nil { + hour, err := strconv.Atoi(date[11:13]) + if err == nil { + min, err := strconv.Atoi(date[14:16]) + if err == nil { + sec, err := strconv.Atoi(date[17:19]) + if err == nil { + seconds := time.Date(year, time.Month(month), day, hour, min, sec, + 0, time.Local).Unix() + return ×tamp.Timestamp{Seconds: seconds, Nanos: 0}, nil + } + } + } + } + } + } + return nil, err +} + +func IsResourceNameValid(name string) bool { + if len(name) > ResourceNameLength { + return false + } + + var validResourceName = regexp.MustCompile(ResourceNamePattern).MatchString + return validResourceName(name) +} diff --git a/pkg/zfssarest/zfssa_fs.go b/pkg/zfssarest/zfssa_fs.go new file mode 100644 index 0000000..922cc86 --- /dev/null +++ b/pkg/zfssarest/zfssa_fs.go @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "context" + "fmt" + "strconv" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" + "net/http" +) + +type Filesystem struct { + MountPoint string `json:"mountpoint"` + CreationTime string `json:"creation"` + RootUser string `json:"root_user"` + RootGroup string `json:"root_group"` + RootPermissions string `json:"root_permissions"` + RestrictChown bool `json:"rstchown"` + ShareNFS string `json:"sharenfs"` + ShareSMB string `json:"sharesmb"` + SpaceData int64 `json:"space_data"` + CanonicalName string `json:"canonical_name"` + RecordSize int64 `json:"recordsize"` + SpaceAvailable int64 `json:"space_available"` + Quota int64 `json:"quota"` + UTF8Only bool `json:"utf8only"` + MaxBlockSize int64 `json:"maxblocksize"` + Atime bool `json:"atime"` + ReadOnly bool `json:"readonly"` + Pool string `json:"pool"` + Name string `json:"name"` + SpaceTotal int64 `json:"space_total"` + SpaceUnused int64 `json:"space_unused_res"` + Project string `json:"project"` + Href string `json:"href"` +} + +type filesystemJSON struct { + FileSystem Filesystem `json:"filesystem"` +} + +type filesystems struct { + List []Filesystem `json:"filesystems"` +} + +// This variable provides a mapping between the name of the parameters used in the storage +// class yaml file and the name of equivalent parameters of the appliance. +var yml2fsProperty = map[string]string { + "rootUser":"root_user", + "rootGroup":"root_group", + "rootPermissions":"root_permissions", + "shareNFS":"sharenfs", + "restrictChown":"rstchown", +} + +func CreateFilesystem(ctx context.Context, token *Token, fsname string, volSize int64, + parameters *map[string]string) (*Filesystem, int, error) { + + pool := (*parameters)["pool"] + project := (*parameters)["project"] + url := fmt.Sprintf(zFilesystems, token.Name, pool, project) + reqBody := buildFilesystemReq(ctx, fsname, volSize, parameters) + rspBody := new(filesystemJSON) + + _, code, err := MakeRequest(ctx, token, "POST", url, reqBody, http.StatusCreated, rspBody) + if err != nil { + return nil, code, err + } + + utils.GetLogCTRL(ctx, 5).Println("CreateFilesystem succeeded") // *** + + return &rspBody.FileSystem, code, nil +} + +// Builds the body of the "create file system" request +func buildFilesystemReq(ctx context.Context, name string, volSize int64, + parameters *map[string]string) *map[string]interface{} { + + fsReq := make(map[string]interface{}) + + fsReq["name"] = name + fsReq["quota"] = volSize + fsReq["reservation"] = volSize + + for key, param := range *parameters { + fsProp, ok := yml2fsProperty[key] + if ok { + switch fsProp { + case "rstchown": + val, err := strconv.ParseBool(param) + if err != nil { + utils.GetLogREST(ctx, 2).Println("Invalid restrict chown, will use default: true", + "rstchown", param) + val = true + } + fsReq[fsProp] = val + default: + fsReq[fsProp] = param + } + utils.GetLogREST(ctx, 5).Println("BuildFSRequest", "key", key, "value", param) + } + } + + return &fsReq +} + +func GetFilesystem(ctx context.Context, token *Token, pool, project, filesystem string) ( + *Filesystem, int, error) { + + url := fmt.Sprintf(zFilesystem, token.Name, pool, project, filesystem) + + rspJSON := &filesystemJSON{} + _, httpStatus, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, rspJSON) + if err != nil { + return nil, httpStatus, err + } + + return &rspJSON.FileSystem, httpStatus, nil +} + +func ModifyFilesystem(ctx context.Context, token *Token, href string, + parameters *map[string]interface{}) (*Filesystem, int, error) { + + url := fmt.Sprintf(zAppliance + href, token.Name) + + rspJSON := &filesystemJSON{} + _, httpStatus, err := MakeRequest(ctx, token, "PUT", url, parameters, http.StatusAccepted, rspJSON) + if err != nil { + return nil, httpStatus, err + } + + return &rspJSON.FileSystem, httpStatus, nil +} + +func DeleteFilesystem(ctx context.Context, token *Token, hRef string) (bool, int, error) { + + utils.GetLogREST(ctx, 5).Println("DeleteFilesystem", "appliance", token.Name, "Filesystem", hRef) + + url := fmt.Sprintf(zAppliance + hRef, token.Name) + + _, httpStatus, err := MakeRequest(ctx, token, "DELETE", url, nil, http.StatusNoContent, nil) + if err != nil { + return false, httpStatus, err + } + + if httpStatus != http.StatusNoContent { + utils.GetLogREST(ctx, 5).Println("DeleteFilesystem", "http status", httpStatus) + if httpStatus >= 200 && httpStatus < 300 { + return false, httpStatus, nil + } + } + + return true, httpStatus, nil +} + +// Returns the List of filesystems associated with the pool and project passed in. To +// get a system wide List of file systems, the pool AND the project must be 'nil' +func GetFilesystems(ctx context.Context, token *Token, pool, project string) ([]Filesystem, error) { + + var url string + if pool != "" && project != "" { + url = fmt.Sprintf(zFilesystems, token.Name, pool, project) + } else if pool == "" && project == "" { + url = fmt.Sprintf(zAllFilesystems, token.Name) + } else { + return nil, grpcStatus.Error(codes.InvalidArgument, "pool and project must be both nil or both not nil") + } + + filesystems := new(filesystems) + filesystems.List = make([]Filesystem, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, filesystems) + if err != nil { + return nil, err + } + + return filesystems.List, nil +} + +func (l *filesystems) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} + +func CloneFileSystemSnapshot(ctx context.Context, token *Token, hRef string, + parameters map[string]interface{}) (*Filesystem, int, error) { + + url := fmt.Sprintf(zAppliance + hRef + "/clone", token.Name) + + rspBody := new(filesystemJSON) + + _, code, err := MakeRequest(ctx, token, "PUT", url, ¶meters, http.StatusCreated, rspBody) + if err != nil { + return nil, code, err + } + + return &rspBody.FileSystem, code, nil +} diff --git a/pkg/zfssarest/zfssa_lun.go b/pkg/zfssarest/zfssa_lun.go new file mode 100644 index 0000000..f881c2c --- /dev/null +++ b/pkg/zfssarest/zfssa_lun.go @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" +) + +const ( + DefaultBlockSize int = 8192 + DevicePathKey string = "devicePath" + MaskAll string = "com.sun.ms.vss.hg.maskAll" +) + +type Lun struct { + SpaceData float64 `json:"space_data"` + CanonicalName string `json:"canonical_name"` + VolumeSize float64 `json:"volsize"` + VolumeBlockSize int64 `json:"volblocksize"` + Pool string `json:"pool"` + Project string `json:"project"` + Name string `json:"name"` + Href string `json:"href"` + AssignedNumber []int32 `json:"assignednumber"` + InitiatorGroup []string `json:"initiatorgroup"` + TargetGroup string `json:"targetgroup"` +} + +type LunJson struct { + LUN Lun `json:"lun"` +} + +type Luns struct { + List []Lun `json:"luns"` +} + +type LunInitiatorGrps struct { + InitiatorGroup []string `json:"initiatorgroup"` +} + +type createLunRequest struct { + Name string `json:"name"` + Size int64 `json:"volsize"` + Blocksize int `json:"volblocksize"` + TargetGroup string `json:"targetgroup"` + Sparse bool `json:"sparse"` + InitiatorGroup []string `json:"initiatorgroup"` +} + +func CreateLUN(ctx context.Context, token *Token, lunName string, volSize int64, + parameters *map[string]string) (*utils.VolumeId, *Lun, int, error) { + + pool := (*parameters)["pool"] + project := (*parameters)["project"] + + url := fmt.Sprintf(zLUNs, token.Name, pool, project) + + blockSizeString := (*parameters)["blockSize"] + blockSize, err := strconv.Atoi(blockSizeString) + if err != nil { + utils.GetLogREST(ctx, 2).Println("Invalid block size, will use default", + "requested_block_size", blockSizeString, "default_block_size", DefaultBlockSize) + blockSize = DefaultBlockSize + } + + volTypeString := (*parameters)["volumeType"] + sparse := false + if volTypeString == "thin" { + sparse = true + } + + reqBody := createLunRequest{ + Name: lunName, + Size: volSize, + Blocksize: blockSize, + TargetGroup: (*parameters)["targetGroup"], + Sparse: sparse, + InitiatorGroup: []string{MaskAll}, + } + + rspBody := &LunJson{} + _, code, err := MakeRequest(ctx, token, "POST", url, reqBody, http.StatusCreated, rspBody) + if err != nil { + return nil, nil, code, err + } + + volumeId := utils.NewVolumeId(utils.BlockVolume, token.Name, pool, project, lunName) + + return volumeId, &rspBody.LUN, code, nil +} + +func GetLun(ctx context.Context, token *Token, pool, project, lun string) (*Lun, int, error) { + + url := fmt.Sprintf(zLUN, token.Name, pool, project, lun) + + rspJSON := &LunJson{} + _, httpStatus, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, rspJSON) + if err != nil { + return nil, httpStatus, err + } + + return &rspJSON.LUN, httpStatus, nil +} + +// Returns the List of LUNs belonging to the pool and project passed in. To +// get a system wide List of LUNs, the pool AND the project must be 'nil' +func GetLuns(ctx context.Context, token *Token, pool, project string) ([]Lun, error) { + + var url string + if pool != "" && project != "" { + url = fmt.Sprintf(zLUNs, token.Name, pool, project) + } else if pool == "" && project == "" { + url = fmt.Sprintf(zAllLUNs, token.Name) + } else { + return nil, grpcStatus.Error(codes.InvalidArgument, "pool and project must be both nil or both not nil") + } + + luns := new(Luns) + luns.List = make([]Lun, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, luns) + if err != nil { + return nil, err + } + + return luns.List, nil +} + +func DeleteLun(ctx context.Context, token *Token, pool, project, lun string) (bool, int, error) { + + url := fmt.Sprintf(zLUN, token.Name, pool, project, lun) + + _, httpStatus, err := MakeRequest(ctx, token, "DELETE", url, nil, http.StatusNoContent, nil) + if err != nil { + return false, httpStatus, err + } + + if httpStatus != http.StatusNoContent { + utils.GetLogREST(ctx, 5).Println("DeleteLun", "http status", httpStatus) + if httpStatus >= 200 && httpStatus < 300 { + return false, httpStatus, nil + } + } + + return true, httpStatus, nil +} + +func GetInitiatorGroupList(ctx context.Context, token *Token, pool, project, + lun string) ([]string, error) { + + url := fmt.Sprintf(zLUN, token.Name, pool, project, lun) + + rspBody := &LunJson{} + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, rspBody) + if err != nil { + return nil, err + } + utils.GetLogREST(ctx, 2).Printf("Retrieved the initiator list: {}", rspBody) + + return rspBody.LUN.InitiatorGroup, nil +} + +func SetInitiatorGroupList(ctx context.Context, token *Token, pool, project, lun, + group string) (int, error) { + + url := fmt.Sprintf(zLUN, token.Name, pool, project, lun) + + reqBody := &LunInitiatorGrps{InitiatorGroup: []string{group}} + utils.GetLogREST(ctx, 2).Printf("Setting up initiator list: {}", reqBody) + _, code, err := MakeRequest(ctx, token, "PUT", url, reqBody, http.StatusAccepted, nil) + return code, err +} + +func (l *Luns) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} + +func CloneLunSnapshot(ctx context.Context, token *Token, hRef string, + parameters map[string]interface{}) (*Lun, int, error) { + + url := fmt.Sprintf(zAppliance + hRef + "/clone", token.Name) + + rspBody := new(LunJson) + + _, code, err := MakeRequest(ctx, token, "PUT", url, ¶meters, http.StatusCreated, rspBody) + if err != nil { + return nil, code, err + } + + return &rspBody.LUN, code, nil +} diff --git a/pkg/zfssarest/zfssa_pool.go b/pkg/zfssarest/zfssa_pool.go new file mode 100644 index 0000000..c06ff0a --- /dev/null +++ b/pkg/zfssarest/zfssa_pool.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "context" + "fmt" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" + "net/http" +) + +type Pool struct { + Status string `json:"status"` + Name string `json:"name"` + Usage struct { + Available int64 `json:"available"` + UsageSnapshots int64 `json:"usage_snapshots"` + UsageMetasize int64 `json:"usage_metasize"` + Used int64 `json:"used"` + UsageChildReservation int64 `json:"usage_child_reservation"` + UsageReplication int64 `json:"usage_replication"` + UsageReservation int64 `json:"usage_reservation"` + Free int64 `json:"free"` + UsageTotal int64 `json:"usage_total"` + UsageMetaused int64 `json:"usage_metaused"` + Total int64 `json:"total"` + UsageData int64 `json:"usage_data"` + } `json:"usage"` + HRef string `json:"href"` + ASN string `json:"asn"` +} + +type poolJSON struct { + Pool Pool `json:"pool"` +} + +type pools struct { + List []Pool `json:"pools"` +} + +func GetPool(ctx context.Context, token *Token, name string) (*Pool, error) { + + // We retrieve the information from the ZFSSA + url := fmt.Sprintf(zPool, token.Name, name) + + json := new(poolJSON) + _, httpstatus, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, json) + if err == nil && httpstatus == http.StatusOK { + return &json.Pool, nil + } + + if err != nil { + utils.GetLogREST(ctx, 2).Println("Request for pool information failed with a local error", + "url", url, "error", err.Error()) + } else { + utils.GetLogREST(ctx, 2).Println("Request for pool information failed with a ZFSSA error", + "url", url, "http status", httpstatus) + } + + return nil, grpcStatus.Error(codes.NotFound,"Pool not found") +} + +func GetPools(ctx context.Context, token *Token) (*[]Pool, error) { + + url := fmt.Sprintf(zPools, token.Name) + + zfssaPools := new(pools) + zfssaPools.List = make([]Pool, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, zfssaPools) + if err != nil { + return nil, err + } + + return &zfssaPools.List, nil +} + +func (l *pools) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} diff --git a/pkg/zfssarest/zfssa_project.go b/pkg/zfssarest/zfssa_project.go new file mode 100644 index 0000000..c4eec75 --- /dev/null +++ b/pkg/zfssarest/zfssa_project.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "context" + "fmt" + "net/http" +) + +type Project struct { + Name string `json:"name"` + Pool string `json:"pool"` + SpaceAvailable int64 `json:"space_available"` +} + +type ProjectJSON struct { + Project Project `json:"project"` +} + +type projects struct { + list []Project `json:"projects"` +} + +func GetProject(ctx context.Context, token *Token, pool string, project string) (*Project, error) { + + url := fmt.Sprintf(zProject, token.Name, pool, project) + + jsonData := &ProjectJSON{} + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, jsonData) + if err != nil { + return nil, err + } + + return &jsonData.Project, nil +} + +// Returns the List of filesystems associated with the pool and project passed in. To +// get a system wide List of file systems, the pool must be 'nil' +func GetProjects(ctx context.Context, token *Token, pool string) ([]Project, error) { + + var url string + if pool != "" { + url = fmt.Sprintf(zProjects, token.Name, pool) + } else { + url = fmt.Sprintf(zAllProjects, token.Name) + } + + projects := new(projects) + projects.list = make([]Project, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, projects) + if err != nil { + return nil, err + } + + return projects.list, nil +} + +func (l *projects) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.list) +} diff --git a/pkg/zfssarest/zfssa_rest.go b/pkg/zfssarest/zfssa_rest.go new file mode 100644 index 0000000..46d5c13 --- /dev/null +++ b/pkg/zfssarest/zfssa_rest.go @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "sync" + "time" + + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" +) + +// Use RESTapi v2 as it returns scriptable and consistent values +const ( + zAppliance string = "https://%s:215" + zServices = zAppliance + "/api/access/v2" + zStorage = zAppliance + "/api/storage/v2" + zSan = zAppliance + "/api/san/v2" + zPools = zStorage + "/pools" + zPool = zPools + "/%s" + zAllProjects = zStorage + "/projects" + zProjects = zPool + "/projects" + zProject = zProjects + "/%s" + zAllFilesystems = zStorage + "/filesystems" + zFilesystems = zProject + "/filesystems" + zFilesystem = zFilesystems + "/%s" + zAllLUNs = zStorage + "/luns" + zLUNs = zProject + "/luns" + zLUN = zLUNs + "/%s" + zAllSnapshots = zStorage + "/snapshots" + zSnapshots = zProject + "/snapshots" + zSnapshot = zSnapshots + "/%s" + zFilesystemSnapshots = zFilesystem + "/snapshots" + zFilesystemSnapshot = zFilesystemSnapshots + "/%s" + zCloneFilesystemSnapshot = zFilesystemSnapshot + "/clone" + zLUNSnapshots = zLUN + "/snapshots" + zLUNSnapshot = zLUNSnapshots + "/%s" + zFilesystemDependents = zFilesystemSnapshot + "/dependents" + zLUNDependents = zLUNSnapshot + "/dependents" + zTargetGroups = zSan + "/%s/target-groups" + zTargetGroup = zTargetGroups + "/%s" + zProperties = zAppliance + "/api/storage/v2/schema" + zProperty = zProperties + "/%s" +) + +// State of a ZFSSA token +const ( + zfssaTokenInvalid = iota + zfssaTokenCreating + zfssaTokenValid +) + +type Token struct { + Name string + cv *sync.Cond + mtx sync.Mutex + user string + password string + state int + xAuthSession string + xAuthName string +} + +type tokenList struct { + mtx sync.Mutex + list map[string]*Token +} + +type faultInfo struct { + Message string `json:"message"` + Code int `json:"code"` + Name string `json:"Name"` +} + +type faultResponse struct { + Fault faultInfo `json:"fault"` +} + +var httpTransport = http.Transport{TLSClientConfig: &tls.Config{}} +var httpClient = &http.Client{Transport: &httpTransport} +var zServicesURL string +var zName string +var tokens tokenList + +// Initializes the ZFSSA REST API interface +// +func InitREST(name string, certs []byte, secure bool) error { + + if secure { + // set TLSv1.2 for the minimum version of supporting TLS + httpTransport.TLSClientConfig.MinVersion = tls.VersionTLS12 + + // Get the SystemCertPool, continue with an empty pool on error + httpTransport.TLSClientConfig.RootCAs, _ = x509.SystemCertPool() + if httpTransport.TLSClientConfig.RootCAs == nil { + httpTransport.TLSClientConfig.RootCAs = x509.NewCertPool() + } + + if ok := httpTransport.TLSClientConfig.RootCAs.AppendCertsFromPEM(certs); !ok { + return errors.New("failed to append the certificate") + } + } + + httpTransport.TLSClientConfig.InsecureSkipVerify = !secure + httpTransport.MaxConnsPerHost = 16 + httpTransport.MaxIdleConnsPerHost = 16 + httpTransport.IdleConnTimeout = 30 * time.Second + + tokens.list = make(map[string]*Token) + zServicesURL = fmt.Sprintf(zServices, name) + zName = name + + return nil +} + +// Looks up a token context based on the user name passed in. If one doesn't exist +// yet, it is created. +func LookUpToken(user, password string) *Token { + + tokens.mtx.Lock() + if token, ok := tokens.list[user]; ok { + tokens.mtx.Unlock() + return token + } + + token := new(Token) + token.Name = zName + token.user = user + token.password = password + token.state = zfssaTokenInvalid + token.xAuthName = "" + token.xAuthSession = "" + token.cv = sync.NewCond(&token.mtx) + + tokens.list[user] = token + tokens.mtx.Unlock() + return token +} + +// Returns a token. If no token is available it attempts to create one. If a previous +// token is passed in, it assumes that the caller received a status 401 from the ZFSSA +// (probably because the token has expired). In that case this function will try to +// create another one or, if another thread is already in the process of creating one, +// it will wait until the creation has completed. +// +// The possible return values are: +// +// Code Message X-Auth-Session +// +// nil Valid +// codes.Internal "Failure getting token" "" +// codes.Internal "Failure creating token" "" +// +// In case of failure, the message logged will provide more information +// as to where the problem occurred. +func getToken(ctx context.Context, token *Token, previous *string) (string, error) { + + token.mtx.Lock() + for { + switch token.state { + case zfssaTokenInvalid: + // No token available. We create one. + token.state = zfssaTokenCreating + token.mtx.Unlock() + + var err error + token.xAuthSession, token.xAuthName, err = createToken(ctx, token) + xAuthSession := token.xAuthSession + + token.mtx.Lock() + if err != nil { + token.state = zfssaTokenInvalid + } else { + token.state = zfssaTokenValid + } + token.cv.Broadcast() + token.mtx.Unlock() + return xAuthSession, err + + case zfssaTokenCreating: + // Another thread is creating a token. We wait until it's done. + token.cv.Wait() + continue + + case zfssaTokenValid: + // We can use the current token. + if previous == nil || *previous != token.xAuthSession { + xAuthSession := token.xAuthSession + token.mtx.Unlock() + return xAuthSession, nil + } + token.state = zfssaTokenInvalid + continue + + default: + panic(fmt.Sprintf("State of token is unknown %s, %d", token.user, token.state)) + } + } +} + +// Send an HTTP request to the ZFSSA to create a non-persistent token. +// +// A non-persistent token is specific to the cluster node on which the ID was +// created and is not synchronized between the cluster peers. +func createToken(ctx context.Context, token *Token) (string, string, error) { + + httpReq, err := http.NewRequest("POST", zServicesURL, bytes.NewBuffer(nil)) + if err != nil { + utils.GetLogREST(ctx,2).Println("Could not build a request to create a token", + "method", "POST", "url", zServicesURL, "error", err.Error()) + return "", "", grpcStatus.Error(codes.Internal, "Failure creating token") + } + + httpReq.Header.Add("X-Auth-User", token.user) + httpReq.Header.Add("X-Auth-Key", token.password) + + httpRsp, err := httpClient.Do(httpReq) + if err != nil { + utils.GetLogREST(ctx,2).Println("Token creation failed in Do", + "url", zServicesURL, "error", err.Error()) + return "", "", grpcStatus.Error(codes.Internal, "Failure creating token") + } + + defer httpRsp.Body.Close() + + if httpRsp.StatusCode != http.StatusCreated { + utils.GetLogREST(ctx,2).Println("Token creation failed in ZFSSA", + "url", zServicesURL, "StatusCode", httpRsp.StatusCode) + return "", "", grpcStatus.Error(codes.Internal, "Failure creating token") + } + + return httpRsp.Header.Get("X-Auth-Session"), httpRsp.Header.Get("X-Auth-Name"), nil +} + +// Makes a request to a target appliance updating the token if needed. +func MakeRequest(ctx context.Context, token *Token, method, url string, reqbody interface{}, status int, + rspbody interface{}) (interface{}, int, error) { + + rsp, code, err := makeRequest(ctx, token, method, url, reqbody, status, rspbody) + if code == http.StatusUnauthorized && err == nil { + rsp, code, err = makeRequest(ctx, token, method, url, reqbody, status, rspbody) + } + return rsp, code, err +} + +// Local function makes the actual request to the ZFSSA. +func makeRequest(ctx context.Context, token *Token, method, url string, reqbody interface{}, status int, + rspbody interface{}) (interface{}, int, error) { + + utils.GetLogREST(ctx,5).Println("MakeRequest to ZFSSA", + "method", method, "url", url, "body", reqbody) + + xAuthSession, err := getToken(ctx, token, nil) + if err != nil { + return nil, 0, err + } + + reqjson, err := json.Marshal(reqbody) + if err != nil { + utils.GetLogREST(ctx,2).Println("json.Marshal call failed", + "method", method, "url", url, "body", reqbody, "error", err.Error()) + return nil, 0, grpcStatus.Error(codes.Unknown, "json.Marshal call failed") + } + + reqhttp, err := http.NewRequest(method, url, bytes.NewBuffer(reqjson)) + if err != nil { + utils.GetLogREST(ctx,2).Println("http.NewRequest call failed", + "method", method, "url", url, "body", reqbody, "error", err.Error()) + return nil, 0, grpcStatus.Error(codes.Unknown, "http.NewRequest call failed") + } + + reqhttp.Header.Add("X-Auth-Session", xAuthSession) + reqhttp.Header.Set("Content-Type", "application/json") + reqhttp.Header.Set("Accept", "application/json") + + rsphttp, err := httpClient.Do(reqhttp) + if err != nil { + utils.GetLogREST(ctx,2).Println("client.do call failed", + "method", method, "url", url, "error", err.Error()) + return nil, 0, grpcStatus.Error(codes.Unknown, "client.do call failed") + } + + // when err is nil, response body is always non-nil + defer rsphttp.Body.Close() + + //d := json.NewDecoder(rsphttp.Body) + //err = d.Decode(rspbody) + + // read json http response + rspjson, err := ioutil.ReadAll(rsphttp.Body) + if err != nil { + utils.GetLogREST(ctx,2).Println("ioutil.ReadAll call failed", + "method", method, "url", url, "code", rsphttp.StatusCode, + "status", rsphttp.Status, "error", err.Error()) + return nil, rsphttp.StatusCode, grpcStatus.Error(codes.Unknown,"ioutil.ReadAll call failed") + } + + if rsphttp.StatusCode == status { + if rspbody != nil { + err = json.Unmarshal(rspjson, rspbody) + if err != nil { + utils.GetLogREST(ctx,2).Println("json.Unmarshal call failed", + "\nmethod", method, "\nurl", url, "\ncode", rsphttp.StatusCode, + "\nstatus", rsphttp.Status, "\nbody", rspjson, "\nerror", err) + return nil, rsphttp.StatusCode, grpcStatus.Error(codes.Unknown, "json.Unmarshal call failed") + } + } + utils.GetLogREST(ctx,5).Println("Successful response from ZFSSA", + "method", method, "url", url, "result", rsphttp.StatusCode) + return rspbody, rsphttp.StatusCode, nil + } + + // We check here whether the token may have expired and renew it if needed. + if rsphttp.StatusCode == http.StatusUnauthorized { + _, err = getToken(ctx, token, &xAuthSession) + return nil, http.StatusUnauthorized, err + } + + // status code was not what the user expected, attempt to unpack + utils.GetLogREST(ctx,2).Println("MakeRequest to ZFSSA resulted in an unexpected status", + "method", method, "url", url, "expected", status, "code", rsphttp.StatusCode, + "status", rsphttp.Status) + + failure := &faultResponse{} + err = json.Unmarshal(rspjson, failure) + var responseString string + if err != nil { + utils.GetLogREST(ctx,2).Println("Failure from ZFSSA could not be un-marshalled", + "method", method, "url", url, "code", rsphttp.StatusCode, + "status", rsphttp.Status, "body", rspjson) + responseString = string(rspjson) + } else { + responseString = failure.Fault.Message + } + + switch rsphttp.StatusCode { + case http.StatusNotFound: + err = grpcStatus.Errorf(codes.NotFound, "Resource not found on target appliance: %s", responseString) + default: + err = grpcStatus.Errorf(codes.Unknown, "Unknown Error Occurred on target appliance: %s", responseString) + } + + return nil, rsphttp.StatusCode, err +} + +type services struct { + List []Service `json:"services"` +} + +type Service struct { + Version string `json:"version"` + Name string `json:"name"` + URI string `json:"uri"` +} + +func GetServices(ctx context.Context, token *Token) (*[]Service, error) { + + rspJSON := new(services) + rspJSON.List = make([]Service, 0) + _, _, err := MakeRequest(ctx, token, "GET", zServicesURL, nil, http.StatusOK, rspJSON) + if err != nil { + return nil, err + } + + return &rspJSON.List, nil +} + +// Unmarshalling of a "List" structure. This structure is the ZFSSA response to +// the http request: +// +// GET /api/access/v1 HTTP/1.1 +// Host: zfs-storage.example.com +// X-Auth-User: admin +// X-Auth-Key: password +// +func (l *services) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} + +// Unmarshalling of a List sent by the ZFSSA +// +func zfssaUnmarshalList(b []byte, l interface{}) error { + + // 'b' starts and ends like this: + // {List:[{...},...,{...}]} + b = b[0:len(b) - 1] + for i := 1; i < len(b); i++ { + if b[i] == '[' { + b = b[i:] + break + } + } + // Now 'b' starts and ends like this: + // [{...},...,{...}] + err := json.Unmarshal(b, l) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/zfssarest/zfssa_san.go b/pkg/zfssarest/zfssa_san.go new file mode 100644 index 0000000..aef9ab0 --- /dev/null +++ b/pkg/zfssarest/zfssa_san.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "context" + "fmt" + "net/http" +) + +type Target struct { + Alias string + Auth string + HRef string + Interfaces []string + IQN string + TargetChapSecret string + TargetChapUser string +} + +type targetJSON struct { + Target Target `json:"Target"` +} + +type TargetGroup struct { + Name string + Targets []string `json:"targets"` +} + +type targetGroupJSON struct { + Group TargetGroup `json:"group"` +} + +func GetTargetGroup(ctx context.Context, token *Token, protocol, groupName string) (*TargetGroup, error) { + + url := fmt.Sprintf(zTargetGroup, token.Name, protocol, groupName) + + rspBody := &targetGroupJSON{} + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, rspBody) + if err != nil { + return nil, err + } + + return &rspBody.Group, nil +} diff --git a/pkg/zfssarest/zfssa_schema.go b/pkg/zfssarest/zfssa_schema.go new file mode 100644 index 0000000..161f4da --- /dev/null +++ b/pkg/zfssarest/zfssa_schema.go @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "context" + "fmt" + "net/http" +) + +type Schema struct { + Type string `json:"type"` + Description string `json:"description"` + Property string `json:"property"` + Href string `json:"href,omitempty"` +} + +// This type is to help deserialize from a ZFSSA response, not used internally +type SchemaList struct { + Schema []Schema `json:"schema"` +} + +type Property struct { + Property Schema `json:"property"` +} + +func CreateProperty(ctx context.Context, token *Token, s Schema) (*Schema, error) { + + utils.GetLogREST(ctx, 5).Println("CreateSchema", "schema", s, "target", token.Name) + + url := fmt.Sprintf(zProperties, token.Name) + + resultSchema := &Property{} + _, _, err := MakeRequest(nil, token, "POST", url, s, http.StatusCreated, resultSchema) + if err != nil { + return nil, err + } + + return &resultSchema.Property, nil +} + +func GetProperty(ctx context.Context, token *Token, property string) (*Schema, error) { + utils.GetLogREST(ctx, 5).Println("GetSchema", "property", property, "target", token.Name) + url := fmt.Sprintf(zProperty, token.Name, property) + + resultSchema := &Property{} + _, _, err := MakeRequest(nil, token, "GET", url, nil, http.StatusOK, resultSchema) + if err != nil { + return nil, err + } + + return &resultSchema.Property, nil +} + +func GetSchema(ctx context.Context, token *Token) (*SchemaList, error) { + utils.GetLogREST(ctx, 5).Println("GetSchema") + + url := fmt.Sprintf(zProperties, token.Name) + + jsonData := &SchemaList{} + _, _, err := MakeRequest(nil, token, "GET", url, nil, http.StatusOK, jsonData) + if err != nil { + return nil, err + } + + return jsonData, nil +} diff --git a/pkg/zfssarest/zfssa_snapshots.go b/pkg/zfssarest/zfssa_snapshots.go new file mode 100644 index 0000000..57824e1 --- /dev/null +++ b/pkg/zfssarest/zfssa_snapshots.go @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ + +package zfssarest + +import ( + "github.com/oracle/zfssa-csi-driver/pkg/utils" + "context" + "fmt" + "net/http" +) + +type Snapshot struct { + Name string `json:"name"` + NumClones int `json:"numclones"` + Creation string `json:"creation"` + Collection string `json:"collection"` + Project string `json:"project"` + CanonicalName string `json:"canonical_name"` + SpaceUnique int64 `json:"space_unique"` + SpaceData int64 `json:"space_data"` + Type string `json:"type"` + ID string `json:"id"` + Pool string `json:"pool"` + Href string `json:"href"` +} + +type snapshotJSON struct { + Snapshot Snapshot `json:"snapshot"` +} + +type snapshots struct { + List []Snapshot `json:"snapshots"` +} + +type Dependent struct { + Project string `json:"project"` + HRef string `json:"href"` + Share string `json:"share"` +} + +type dependents struct { + List []Dependent `json:"dependents"` +} + +// Issues a request to the appliance to create a snapshot. The source of the snapshot, LUN +// or filesystem, is determine by the HREF passed in. +func CreateSnapshot(ctx context.Context, token *Token, href, name string) (*Snapshot, int, error) { + + url := fmt.Sprintf(zAppliance + href + "/snapshots", token.Name) + + reqBody := make(map[string]interface{}) + reqBody["name"] = name + rspBody := new(snapshotJSON) + + _, code, err := MakeRequest(ctx, token, "POST", url, &reqBody, http.StatusCreated, rspBody) + if err != nil { + return nil, code, err + } + + return &rspBody.Snapshot, code, nil +} + +// Issues a request to the appliance to delete a snapshot. The source of the snapshot, LUN +// or filesystem, is determine by the HREF passed in. +func DeleteSnapshot(ctx context.Context, token *Token, href string) ( + bool, int, error) { + + url := fmt.Sprintf(zAppliance + href, token.Name) + + _, httpStatus, err := MakeRequest(ctx, token, "DELETE", url, nil, http.StatusNoContent, nil) + if err != nil { + return false, httpStatus, err + } + + if httpStatus != http.StatusNoContent { + utils.GetLogREST(ctx, 5).Println("DeleteFilesystem", "http status", httpStatus) + if httpStatus >= 200 && httpStatus < 300 { + return false, httpStatus, nil + } + } + + return true, httpStatus, nil +} + +// Issues a request to the appliance asking for the detailed information of a snapshot. +func GetSnapshot(ctx context.Context, token *Token, href, name string) (*Snapshot, int, error) { + + url := fmt.Sprintf(zAppliance + href + "/snapshots/%s", token.Name, name) + + rspJSON := &snapshotJSON{} + _, httpStatus, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, rspJSON) + if err != nil { + return nil, httpStatus, err + } + + return &rspJSON.Snapshot, httpStatus, nil +} + +// Issues a request to the appliance asking for a volume's snapshot list. +func GetSnapshots(ctx context.Context, token *Token, href string) ([]Snapshot, error) { + + var url string + + if len(href) > 0 { + url = fmt.Sprintf(zAppliance + href + "/snapshots", token.Name) + } else { + url = fmt.Sprintf(zAllSnapshots, token.Name) + } + + snapshots := new(snapshots) + snapshots.List = make([]Snapshot, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, snapshots) + if err != nil { + return nil, err + } + + return snapshots.List, nil +} + +// Returns the clones depending on a snapshot. +func GetSnapshotDependents(ctx context.Context, token *Token, href string) (*[]Dependent, error) { + + url := fmt.Sprintf(zAppliance + href + "/dependents", token.Name) + + dependents := new(dependents) + dependents.List = make([]Dependent, 0) + + _, _, err := MakeRequest(ctx, token, "GET", url, nil, http.StatusOK, dependents) + if err != nil { + return nil, err + } + + return &dependents.List, nil +} + +func (l *snapshots) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} + +func (l *dependents) UnmarshalJSON(b []byte) error { + return zfssaUnmarshalList(b, &l.List) +} diff --git a/release-tools/build.make b/release-tools/build.make new file mode 100644 index 0000000..5bc6b98 --- /dev/null +++ b/release-tools/build.make @@ -0,0 +1,51 @@ +# Copyright (c) 2021 Oracle and/or its affiliates. +# +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# + +.PHONY: build-% build container-% container push-% push clean + +# Registry used on push +REGISTRY_NAME= + +# Revision +REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git rev-list -n1 HEAD) + +# A "zfssa-xxx" image gets built if the current branch is "zfssa-xxx". +IMAGE_TAGS=$(shell git branch | grep '* zfssa-' | grep -v -e ' -> ' | sed -e 's/\* //') + +# Images are named after the command contained in them. +IMAGE_NAME=$(REGISTRY_NAME)/$* + +build-%: + mkdir -p bin + CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + +container-%: build-% + docker build -t $*:latest -f $(shell if [ -e ./cmd/zfssa-csi-driver/$*/Dockerfile ]; then echo ./cmd/zfssa-csi-driver/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . --build-arg var_proxy=$(DOCKER_PROXY) + +push-%: container-% + set -ex; \ + push_image () { \ + docker tag $*:latest $(IMAGE_NAME):$$tag; \ + docker push $(IMAGE_NAME):$$tag; \ + }; \ + for tag in $(IMAGE_TAGS); do \ + if [ "$$tag" = "canary" ] || echo "$$tag" | grep -q -e '-canary$$'; then \ + : "creating or overwriting canary image"; \ + push_image; \ + elif docker pull $(IMAGE_NAME):$$tag 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$$tag not found"; then \ + : "creating release image"; \ + push_image; \ + else \ + : "release image $(IMAGE_NAME):$$tag already exists, skipping push"; \ + fi; \ + done + +build: $(CMDS:%=build-%) +container: $(CMDS:%=container-%) +push: $(CMDS:%=push-%) + +clean: + -rm -rf bin + From 05d2a3a084752136a968135fe08d53165a568481 Mon Sep 17 00:00:00 2001 From: "JEONGTAE.KIM@ORACLE.COM" Date: Tue, 24 Aug 2021 16:50:27 -0600 Subject: [PATCH 2/5] Added Dockerfile to build CSI driver container --- Dockerfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e006a11 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# Copyright (c) 2021 Oracle and/or its affiliates. +# +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +# +# This is the Dockerfile for Oracle ZFS Storage Appliance CSI Driver +# + +FROM container-registry.oracle.com/os/oraclelinux:7-slim +LABEL maintainers="Oracle" +LABEL description="Oracle ZFS Storage Appliance CSI Driver for Kubernetes" + +ARG var_proxy +ENV http_proxy=$var_proxy +ENV https_proxy=$var_proxy + +# Add util-linux to get a new version of losetup. +RUN yum -y install iscsi-initiator-utils nfs-utils e2fsprogs xfsprogs && yum clean all + +ENV http_proxy "" +ENV https_proxy "" + +COPY ./bin/zfssa-csi-driver /zfssa-csi-driver +ENTRYPOINT ["/zfssa-csi-driver"] From c87f2d9b27de8af95c1835e4f130e8f9ba4951ba Mon Sep 17 00:00:00 2001 From: "JEONGTAE.KIM@ORACLE.COM" Date: Tue, 24 Aug 2021 16:58:08 -0600 Subject: [PATCH 3/5] Updated char version --- deploy/helm/k8s-1.17/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/k8s-1.17/Chart.yaml b/deploy/helm/k8s-1.17/Chart.yaml index ee1c303..4adea21 100644 --- a/deploy/helm/k8s-1.17/Chart.yaml +++ b/deploy/helm/k8s-1.17/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 name: zfssa-csi -version: 0.0.2 +version: 1.0.0 description: Deploys Oracle ZFS Storage Appliance CSI Plugin. From ffa40275f22fa9dc9dfd417baa55b14a2cfeea31 Mon Sep 17 00:00:00 2001 From: Paul Monday Date: Wed, 25 Aug 2021 14:48:23 -0600 Subject: [PATCH 4/5] init: update the README file with information about Oracle Container Registry --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6177d87..61a4b05 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# About zfssa-cs-driver +# About zfssa-csi-driver This plugin supports Oracle ZFS Storage Appliance as a backend for block storage (iSCSI volumes) and file storage (NFS). @@ -19,6 +19,7 @@ ZFS Storage Appiance (or simulator). The management and data path can be the same address. * A suitable container image build environment. The Makefile currently uses docker but with minor updates to release-tools/build.make, podman should also be usable. +* An account for use with [container-registry.oracle.com](https://container-registry.oracle.com/) image registry. ## Unsupported Functionality ZFS Storage Appliance CSI driver does not support the following functionality: @@ -28,9 +29,23 @@ ZFS Storage Appliance CSI driver does not support the following functionality: Use and enhance the Makefile in the root directory and release-tools/build.make. +Build the driver: ``` -make +make build ``` +Depending on your golang installation, there may be dependencies identified by the build, install +these and retry the build. + +Prior to building the container image, docker login to container-registry.oracle.com so the +parent image container-registry.oracle.com/os/oraclelinux:7-slim can be retrieved. There may +be license terms to accept at the web entry to the container registry: +[container-registry.oracle.com](https://container-registry.oracle.com/). + +Once you are logged in you can make the container with the following: +``` +make container +``` +Tag and push the resulting container image to a container registry. ## Installation From ef35bf1a6d87805327c763b57a70eeb7137c8311 Mon Sep 17 00:00:00 2001 From: Paul Monday Date: Wed, 25 Aug 2021 17:08:41 -0600 Subject: [PATCH 5/5] init: update README.md with alternative locations of parent image --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 61a4b05..88b44a6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ ZFS Storage Appiance (or simulator). The management and data path can be the same address. * A suitable container image build environment. The Makefile currently uses docker but with minor updates to release-tools/build.make, podman should also be usable. -* An account for use with [container-registry.oracle.com](https://container-registry.oracle.com/) image registry. ## Unsupported Functionality ZFS Storage Appliance CSI driver does not support the following functionality: @@ -36,16 +35,16 @@ make build Depending on your golang installation, there may be dependencies identified by the build, install these and retry the build. -Prior to building the container image, docker login to container-registry.oracle.com so the -parent image container-registry.oracle.com/os/oraclelinux:7-slim can be retrieved. There may -be license terms to accept at the web entry to the container registry: -[container-registry.oracle.com](https://container-registry.oracle.com/). +The parent image for the container is container-registry.oracle.com/os/oraclelinux:7-slim, refer +to [container-registry.oracle.com](https://container-registry.oracle.com/) for more information. +The parent image can also be obtained from ghcr.io/oracle/oraclelinux and docker.io/library/oraclelinux. -Once you are logged in you can make the container with the following: +To build the container: ``` make container ``` -Tag and push the resulting container image to a container registry. +Tag and push the resulting container image to a container registry available to the +Kubernetes cluster where it will be deployed. ## Installation