From 1b76b0b2ff06d92e0b4b0d3d709effbaf18f9440 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 20 Nov 2017 12:57:24 -0500 Subject: [PATCH 1/4] Allow node to update PVC's status Implement node policy feature gates Add tests for node policy update --- plugin/pkg/admission/noderestriction/BUILD | 3 + .../admission/noderestriction/admission.go | 58 ++++++++++++++++++- plugin/pkg/auth/authorizer/node/BUILD | 2 + .../auth/authorizer/node/node_authorizer.go | 36 +++++++++++- .../authorizer/rbac/bootstrappolicy/policy.go | 15 ++++- test/integration/auth/BUILD | 4 ++ test/integration/auth/node_test.go | 41 +++++++++++++ 7 files changed, 153 insertions(+), 6 deletions(-) diff --git a/plugin/pkg/admission/noderestriction/BUILD b/plugin/pkg/admission/noderestriction/BUILD index f909885429e..8d2777ea41f 100644 --- a/plugin/pkg/admission/noderestriction/BUILD +++ b/plugin/pkg/admission/noderestriction/BUILD @@ -17,11 +17,14 @@ go_library( "//pkg/auth/nodeidentifier:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 71feec8c3d4..d958dad3762 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -23,13 +23,16 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" podutil "k8s.io/kubernetes/pkg/api/pod" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/auth/nodeidentifier" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" coreinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + "k8s.io/kubernetes/pkg/features" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" ) @@ -82,6 +85,7 @@ func (p *nodePlugin) ValidateInitialization() error { var ( podResource = api.Resource("pods") nodeResource = api.Resource("nodes") + pvcResource = api.Resource("persistentvolumeclaims") ) func (c *nodePlugin) Admit(a admission.Attributes) error { @@ -113,6 +117,14 @@ func (c *nodePlugin) Admit(a admission.Attributes) error { case nodeResource: return c.admitNode(nodeName, a) + case pvcResource: + switch a.GetSubresource() { + case "status": + return c.admitPVCStatus(nodeName, a) + default: + return admission.NewForbidden(a, fmt.Errorf("may only update PVC status")) + } + default: return nil } @@ -189,7 +201,7 @@ func (c *nodePlugin) admitPodStatus(nodeName string, a admission.Attributes) err // require an existing pod pod, ok := a.GetOldObject().(*api.Pod) if !ok { - return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject())) + return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetOldObject())) } // only allow a node to update status of a pod bound to itself if pod.Spec.NodeName != nodeName { @@ -241,6 +253,50 @@ func (c *nodePlugin) admitPodEviction(nodeName string, a admission.Attributes) e } } +func (c *nodePlugin) admitPVCStatus(nodeName string, a admission.Attributes) error { + switch a.GetOperation() { + case admission.Update: + if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + return admission.NewForbidden(a, fmt.Errorf("node %q may not update persistentvolumeclaim metadata", nodeName)) + } + + oldPVC, ok := a.GetOldObject().(*api.PersistentVolumeClaim) + if !ok { + return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetOldObject())) + } + + newPVC, ok := a.GetObject().(*api.PersistentVolumeClaim) + if !ok { + return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject())) + } + + // make copies for comparison + oldPVC = oldPVC.DeepCopy() + newPVC = newPVC.DeepCopy() + + // zero out resourceVersion to avoid comparing differences, + // since the new object could leave it empty to indicate an unconditional update + oldPVC.ObjectMeta.ResourceVersion = "" + newPVC.ObjectMeta.ResourceVersion = "" + + oldPVC.Status.Capacity = nil + newPVC.Status.Capacity = nil + + oldPVC.Status.Conditions = nil + newPVC.Status.Conditions = nil + + // ensure no metadata changed. nodes should not be able to relabel, add finalizers/owners, etc + if !apiequality.Semantic.DeepEqual(oldPVC, newPVC) { + return admission.NewForbidden(a, fmt.Errorf("node %q may not update fields other than status.capacity and status.conditions: %v", nodeName, diff.ObjectReflectDiff(oldPVC, newPVC))) + } + + return nil + + default: + return admission.NewForbidden(a, fmt.Errorf("unexpected operation %q", a.GetOperation())) + } +} + func (c *nodePlugin) admitNode(nodeName string, a admission.Attributes) error { requestedName := a.GetName() if a.GetOperation() == admission.Create { diff --git a/plugin/pkg/auth/authorizer/node/BUILD b/plugin/pkg/auth/authorizer/node/BUILD index 1dbbdb2458b..46bb915ba8a 100644 --- a/plugin/pkg/auth/authorizer/node/BUILD +++ b/plugin/pkg/auth/authorizer/node/BUILD @@ -36,6 +36,7 @@ go_library( "//pkg/apis/rbac:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", "//pkg/client/informers/informers_generated/internalversion/core/internalversion:go_default_library", + "//pkg/features:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", "//third_party/forked/gonum/graph:go_default_library", "//third_party/forked/gonum/graph/simple:go_default_library", @@ -43,6 +44,7 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", ], ) diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer.go b/plugin/pkg/auth/authorizer/node/node_authorizer.go index 62ab4a5b086..bf728ccc5e6 100644 --- a/plugin/pkg/auth/authorizer/node/node_authorizer.go +++ b/plugin/pkg/auth/authorizer/node/node_authorizer.go @@ -23,9 +23,11 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" + utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" rbacapi "k8s.io/kubernetes/pkg/apis/rbac" "k8s.io/kubernetes/pkg/auth/nodeidentifier" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" "k8s.io/kubernetes/third_party/forked/gonum/graph" "k8s.io/kubernetes/third_party/forked/gonum/graph/traverse" @@ -85,6 +87,11 @@ func (r *NodeAuthorizer) Authorize(attrs authorizer.Attributes) (authorizer.Deci case configMapResource: return r.authorizeGet(nodeName, configMapVertexType, attrs) case pvcResource: + if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + if attrs.GetSubresource() == "status" { + return r.authorizeStatusUpdate(nodeName, pvcVertexType, attrs) + } + } return r.authorizeGet(nodeName, pvcVertexType, attrs) case pvResource: return r.authorizeGet(nodeName, pvVertexType, attrs) @@ -98,17 +105,42 @@ func (r *NodeAuthorizer) Authorize(attrs authorizer.Attributes) (authorizer.Deci return authorizer.DecisionNoOpinion, "", nil } +// authorizeStatusUpdate authorizes get/update/patch requests to status subresources of the specified type if they are related to the specified node +func (r *NodeAuthorizer) authorizeStatusUpdate(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { + switch attrs.GetVerb() { + case "update", "patch": + // ok + default: + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return authorizer.DecisionNoOpinion, "can only get/update/patch this type", nil + } + + if attrs.GetSubresource() != "status" { + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return authorizer.DecisionNoOpinion, "can only update status subresource", nil + } + + return r.authorize(nodeName, startingType, attrs) +} + // authorizeGet authorizes "get" requests to objects of the specified type if they are related to the specified node func (r *NodeAuthorizer) authorizeGet(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { - if attrs.GetVerb() != "get" || len(attrs.GetName()) == 0 { + if attrs.GetVerb() != "get" { glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "can only get individual resources of this type", nil } - if len(attrs.GetSubresource()) > 0 { glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) return authorizer.DecisionNoOpinion, "cannot get subresource", nil } + return r.authorize(nodeName, startingType, attrs) +} + +func (r *NodeAuthorizer) authorize(nodeName string, startingType vertexType, attrs authorizer.Attributes) (authorizer.Decision, string, error) { + if len(attrs.GetName()) == 0 { + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return authorizer.DecisionNoOpinion, "No Object name found", nil + } ok, err := r.hasPathFrom(nodeName, startingType, attrs.GetNamespace(), attrs.GetName()) if err != nil { diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 29bd87f2a76..43a898ae2fa 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -91,7 +91,7 @@ func addClusterRoleBindingLabel(rolebindings []rbac.ClusterRoleBinding) { } func NodeRules() []rbac.PolicyRule { - return []rbac.PolicyRule{ + nodePolicyRules := []rbac.PolicyRule{ // Needed to check API access. These creates are non-mutating rbac.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews", "localsubjectaccessreviews").RuleOrDie(), @@ -123,11 +123,12 @@ func NodeRules() []rbac.PolicyRule { // Needed for imagepullsecrets, rbd/ceph and secret volumes, and secrets in envs // Needed for configmap volume and envs - // Use the NodeRestriction admission plugin to limit a node to get secrets/configmaps referenced by pods bound to itself. + // Use the Node authorization mode to limit a node to get secrets/configmaps referenced by pods bound to itself. rbac.NewRule("get").Groups(legacyGroup).Resources("secrets", "configmaps").RuleOrDie(), // Needed for persistent volumes - // Use the NodeRestriction admission plugin to limit a node to get pv/pvc objects referenced by pods bound to itself. + // Use the Node authorization mode to limit a node to get pv/pvc objects referenced by pods bound to itself. rbac.NewRule("get").Groups(legacyGroup).Resources("persistentvolumeclaims", "persistentvolumes").RuleOrDie(), + // TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node // Needed for glusterfs volumes rbac.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), @@ -135,6 +136,14 @@ func NodeRules() []rbac.PolicyRule { // for it to be signed. This allows the kubelet to rotate it's own certificate. rbac.NewRule("create", "get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), } + + if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + // Use the Node authorization mode to limit a node to update status of pvc objects referenced by pods bound to itself. + // Use the NodeRestriction admission plugin to limit a node to just update the status stanza. + pvcStatusPolicyRule := rbac.NewRule("get", "update", "patch").Groups(legacyGroup).Resources("persistentvolumeclaims/status").RuleOrDie() + nodePolicyRules = append(nodePolicyRules, pvcStatusPolicyRule) + } + return nodePolicyRules } // ClusterRoles returns the cluster roles to bootstrap an API server with diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index c91ff5fd669..ccd4ec402c4 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -32,6 +32,7 @@ go_test( "//pkg/bootstrap/api:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubeapiserver/authorizer:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/rbac/clusterrole:go_default_library", @@ -56,6 +57,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/group:go_default_library", @@ -66,6 +68,8 @@ go_test( "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library", "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 4fab8257654..32a699b12c4 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -17,6 +17,7 @@ limitations under the License. package auth import ( + "fmt" "net/http" "net/http/httptest" "path/filepath" @@ -27,9 +28,12 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/request/bearertoken" "k8s.io/apiserver/pkg/authentication/token/tokenfile" "k8s.io/apiserver/pkg/authentication/user" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" restclient "k8s.io/client-go/rest" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" @@ -37,6 +41,7 @@ import ( "k8s.io/kubernetes/pkg/auth/nodeidentifier" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer" "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/test/integration/framework" @@ -131,6 +136,7 @@ func TestNodeAuthorizer(t *testing.T) { }); err != nil { t.Fatal(err) } + if _, err := superuserClient.Core().PersistentVolumes().Create(&api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{Name: "mypv"}, Spec: api.PersistentVolumeSpec{ @@ -249,6 +255,21 @@ func TestNodeAuthorizer(t *testing.T) { }) } + capacity := 50 + updatePVCCapacity := func(client clientset.Interface) error { + capacity++ + statusString := fmt.Sprintf("{\"status\": {\"capacity\": {\"storage\": \"%dG\"}}}", capacity) + patchBytes := []byte(statusString) + _, err := client.Core().PersistentVolumeClaims("ns").Patch("mypvc", types.StrategicMergePatchType, patchBytes, "status") + return err + } + + updatePVCPhase := func(client clientset.Interface) error { + patchBytes := []byte(`{"status":{"phase": "Bound"}}`) + _, err := client.Core().PersistentVolumeClaims("ns").Patch("mypvc", types.StrategicMergePatchType, patchBytes, "status") + return err + } + nodeanonClient := clientsetForToken(tokenNodeUnknown, clientConfig) node1Client := clientsetForToken(tokenNode1, clientConfig) node2Client := clientsetForToken(tokenNode2, clientConfig) @@ -287,6 +308,7 @@ func TestNodeAuthorizer(t *testing.T) { expectForbidden(t, getConfigMap(node2Client)) expectForbidden(t, getPVC(node2Client)) expectForbidden(t, getPV(node2Client)) + expectForbidden(t, createNode2NormalPod(nodeanonClient)) // mirror pod and self node lifecycle is allowed expectAllowed(t, createNode2MirrorPod(node2Client)) @@ -333,6 +355,7 @@ func TestNodeAuthorizer(t *testing.T) { expectAllowed(t, getConfigMap(node2Client)) expectAllowed(t, getPVC(node2Client)) expectAllowed(t, getPV(node2Client)) + expectForbidden(t, createNode2NormalPod(node2Client)) expectAllowed(t, updateNode2NormalPodStatus(node2Client)) expectAllowed(t, deleteNode2NormalPod(node2Client)) @@ -343,6 +366,24 @@ func TestNodeAuthorizer(t *testing.T) { expectAllowed(t, createNode2MirrorPod(superuserClient)) expectAllowed(t, createNode2NormalPodEviction(node2Client)) expectAllowed(t, createNode2MirrorPodEviction(node2Client)) + + // re-create a pod as an admin to add object references + expectAllowed(t, createNode2NormalPod(superuserClient)) + // With ExpandPersistentVolumes feature disabled + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, false)() + // node->pvc relationship not established + expectForbidden(t, updatePVCCapacity(node1Client)) + // node->pvc relationship established but feature is disabled + expectForbidden(t, updatePVCCapacity(node2Client)) + + //Enabled ExpandPersistentVolumes feature + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExpandPersistentVolumes, true)() + // Node->pvc relationship not established + expectForbidden(t, updatePVCCapacity(node1Client)) + // node->pvc relationship established and feature is enabled + expectAllowed(t, updatePVCCapacity(node2Client)) + // node->pvc relationship established but updating phase + expectForbidden(t, updatePVCPhase(node2Client)) } func expectForbidden(t *testing.T, err error) { From 2f2a6436843151d8c7a251062f09084d2dd7d953 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 20 Nov 2017 12:59:48 -0500 Subject: [PATCH 2/4] Implement file system resizing support on kubelet start Update bazel files Fix operation executor tests --- pkg/kubelet/events/event.go | 2 + pkg/util/BUILD | 1 + pkg/util/mount/mount_linux.go | 6 +- pkg/util/resizefs/BUILD | 38 +++++ pkg/util/resizefs/resizefs_linux.go | 143 ++++++++++++++++++ pkg/util/resizefs/resizefs_unsupported.go | 40 +++++ pkg/volume/util/operationexecutor/BUILD | 2 + .../operation_executor_test.go | 1 - .../operationexecutor/operation_generator.go | 98 ++++++++++++ 9 files changed, 327 insertions(+), 4 deletions(-) create mode 100644 pkg/util/resizefs/BUILD create mode 100644 pkg/util/resizefs/resizefs_linux.go create mode 100644 pkg/util/resizefs/resizefs_unsupported.go diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index 59c5e9c5ba1..12740573811 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -52,6 +52,8 @@ const ( FailedDetachVolume = "FailedDetachVolume" FailedMountVolume = "FailedMount" VolumeResizeFailed = "VolumeResizeFailed" + FileSystemResizeFailed = "FileSystemResizeFailed" + FileSystemResizeSuccess = "FileSystemResizeSuccessful" FailedUnMountVolume = "FailedUnMount" FailedMapVolume = "FailedMapVolume" FailedUnmapDevice = "FailedUnmapDevice" diff --git a/pkg/util/BUILD b/pkg/util/BUILD index ae800c2ce19..362af715259 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -47,6 +47,7 @@ filegroup( "//pkg/util/procfs:all-srcs", "//pkg/util/reflector/prometheus:all-srcs", "//pkg/util/removeall:all-srcs", + "//pkg/util/resizefs:all-srcs", "//pkg/util/resourcecontainer:all-srcs", "//pkg/util/rlimit:all-srcs", "//pkg/util/selinux:all-srcs", diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 9caec6707b6..71064f321e3 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -498,7 +498,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, if mountErr != nil { // Mount failed. This indicates either that the disk is unformatted or // it contains an unexpected filesystem. - existingFormat, err := mounter.getDiskFormat(source) + existingFormat, err := mounter.GetDiskFormat(source) if err != nil { return err } @@ -536,8 +536,8 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, return mountErr } -// getDiskFormat uses 'lsblk' to see if the given disk is unformated -func (mounter *SafeFormatAndMount) getDiskFormat(disk string) (string, error) { +// GetDiskFormat uses 'lsblk' to see if the given disk is unformated +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { args := []string{"-n", "-o", "FSTYPE", disk} glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) dataOut, err := mounter.Exec.Run("lsblk", args...) diff --git a/pkg/util/resizefs/BUILD b/pkg/util/resizefs/BUILD new file mode 100644 index 00000000000..aaa1f9aa5f4 --- /dev/null +++ b/pkg/util/resizefs/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "resizefs_unsupported.go", + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "resizefs_linux.go", + ], + "//conditions:default": [], + }), + importpath = "k8s.io/kubernetes/pkg/util/resizefs", + visibility = ["//visibility:public"], + deps = [ + "//pkg/util/mount:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/util/resizefs/resizefs_linux.go b/pkg/util/resizefs/resizefs_linux.go new file mode 100644 index 00000000000..6a4d82d6c03 --- /dev/null +++ b/pkg/util/resizefs/resizefs_linux.go @@ -0,0 +1,143 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resizefs + +import ( + "fmt" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/mount" + utilexec "k8s.io/utils/exec" +) + +const ( + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 +) + +// ResizeFs Provides support for resizing file systems +type ResizeFs struct { + mounter *mount.SafeFormatAndMount +} + +// NewResizeFs returns new instance of resizer +func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { + return &ResizeFs{mounter: mounter} +} + +// Resize perform resize of file system +func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { + format, err := resizefs.mounter.GetDiskFormat(devicePath) + + if err != nil { + formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err) + return false, formatErr + } + + // If disk has no format, there is no need to resize the disk because mkfs.* + // by default will use whole disk anyways. + if format == "" { + return false, nil + } + + deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath) + + if err != nil { + deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err) + return false, deviceOpenErr + } + + if deviceOpened { + deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath) + return false, deviceAlreadyOpenErr + } + + switch format { + case "ext3", "ext4": + fsckErr := resizefs.extFsck(devicePath, format) + if fsckErr != nil { + return false, fsckErr + } + return resizefs.extResize(devicePath) + case "xfs": + fsckErr := resizefs.fsckDevice(devicePath) + if fsckErr != nil { + return false, fsckErr + } + return resizefs.xfsResize(devicePath) + } + return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath) +} + +func (resizefs *ResizeFs) fsckDevice(devicePath string) error { + glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath) + args := []string{"-a", devicePath} + out, err := resizefs.mounter.Exec.Run("fsck", args...) + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out)) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + glog.Infof("`fsck` error %s", string(out)) + } + } + return nil +} + +func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error { + glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath) + args := []string{"-f", "-y", devicePath} + out, err := resizefs.mounter.Run("fsck."+fsType, args...) + if err != nil { + return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out)) + } + return nil +} + +func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { + output, err := resizefs.mounter.Exec.Run("resize2fs", devicePath) + if err == nil { + glog.V(2).Infof("Device %s resized successfully", devicePath) + return true, nil + } + + resizeError := fmt.Errorf("resize of device %s failed: %v. resize2fs output: %s", devicePath, err, string(output)) + return false, resizeError + +} + +func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) { + args := []string{"-d", devicePath} + output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...) + + if err == nil { + glog.V(2).Infof("Device %s resized successfully", devicePath) + return true, nil + } + + resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output)) + return false, resizeError +} diff --git a/pkg/util/resizefs/resizefs_unsupported.go b/pkg/util/resizefs/resizefs_unsupported.go new file mode 100644 index 00000000000..9241d7d5b27 --- /dev/null +++ b/pkg/util/resizefs/resizefs_unsupported.go @@ -0,0 +1,40 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resizefs + +import ( + "fmt" + + "k8s.io/kubernetes/pkg/util/mount" +) + +// ResizeFs Provides support for resizing file systems +type ResizeFs struct { + mounter *mount.SafeFormatAndMount +} + +// NewResizeFs returns new instance of resizer +func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { + return &ResizeFs{mounter: mounter} +} + +// Resize perform resize of file system +func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { + return false, fmt.Errorf("Resize is not supported for this build") +} diff --git a/pkg/volume/util/operationexecutor/BUILD b/pkg/volume/util/operationexecutor/BUILD index 36eafd136d1..15c2ac27d83 100644 --- a/pkg/volume/util/operationexecutor/BUILD +++ b/pkg/volume/util/operationexecutor/BUILD @@ -18,6 +18,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/util/mount:go_default_library", + "//pkg/util/resizefs:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/nestedpendingoperations:go_default_library", @@ -28,6 +29,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", diff --git a/pkg/volume/util/operationexecutor/operation_executor_test.go b/pkg/volume/util/operationexecutor/operation_executor_test.go index 3624c889156..4e06b39616b 100644 --- a/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -80,7 +80,6 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForAttachablePlugins(t *te volumesToMount := make([]VolumeToMount, numVolumesToAttach) pdName := "pd-volume" volumeName := v1.UniqueVolumeName(pdName) - // Act for i := range volumesToMount { podName := "pod-" + strconv.Itoa((i + 1)) diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 909b3aae745..c99060ceafe 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -17,6 +17,7 @@ limitations under the License. package operationexecutor import ( + "encoding/json" "fmt" "strings" "time" @@ -26,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" @@ -33,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/features" kevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/util/resizefs" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" @@ -451,6 +454,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + mounter := og.volumePluginMgr.Host.GetMounter(volumePlugin.GetPluginName()) + resizeError := og.resizeFileSystem(volumeToMount, devicePath, mounter) + + if resizeError != nil { + return volumeToMount.GenerateErrorDetailed("MountVolume.Resize failed", resizeError) + } + deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { @@ -528,6 +538,65 @@ func (og *operationGenerator) GenerateMountVolumeFunc( }, volumePlugin.GetPluginName(), nil } +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, mounter mount.Interface) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) + return nil + } + + // Get expander, if possible + expandableVolumePlugin, _ := + og.volumePluginMgr.FindExpandablePluginBySpec(volumeToMount.VolumeSpec) + + if expandableVolumePlugin != nil && + expandableVolumePlugin.RequiresFSResize() && + volumeToMount.VolumeSpec.PersistentVolume != nil { + pv := volumeToMount.VolumeSpec.PersistentVolume + pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(pv.Spec.ClaimRef.Name, metav1.GetOptions{}) + if err != nil { + // Return error rather than leave the file system un-resized, caller will log and retry + return volumeToMount.GenerateErrorDetailed("MountVolume get PVC failed", err) + } + + pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage] + pvSpecCap := pv.Spec.Capacity[v1.ResourceStorage] + if pvcStatusCap.Cmp(pvSpecCap) < 0 { + // File system resize was requested, proceed + glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + + diskFormatter := &mount.SafeFormatAndMount{ + Interface: mounter, + Exec: og.volumePluginMgr.Host.GetExec(expandableVolumePlugin.GetPluginName()), + } + + resizer := resizefs.NewResizeFs(diskFormatter) + resizeStatus, resizeErr := resizer.Resize(devicePath) + + if resizeErr != nil { + resizeDetailedError := volumeToMount.GenerateErrorDetailed("MountVolume.resizeFileSystem failed", resizeErr) + glog.Error(resizeDetailedError) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, resizeDetailedError.Error()) + return resizeDetailedError + } + + if resizeStatus { + simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem succeeded", "") + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg) + glog.Infof(detailedMsg) + } + + // File system resize succeeded, now update the PVC's Capacity to match the PV's + err = updatePVCStatusCapacity(pvc.Name, pvc, pv.Spec.Capacity, og.kubeClient) + if err != nil { + // On retry, resizeFileSystem will be called again but do nothing + return volumeToMount.GenerateErrorDetailed("MountVolume update PVC status failed", err) + } + return nil + } + } + return nil +} + func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { @@ -1104,6 +1173,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, expandErr.Error()) return expandErr } + glog.Infof("ExpandVolume succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) newSize = updatedSize // k8s doesn't have transactions, we can't guarantee that after updating PV - updating PVC will be // successful, that is why all PVCs for which pvc.Spec.Size > pvc.Status.Size must be reprocessed @@ -1115,6 +1185,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, updateErr.Error()) return updateErr } + glog.Infof("ExpandVolume.UpdatePV succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) } // No Cloudprovider resize needed, lets mark resizing as done @@ -1190,3 +1261,30 @@ func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (boo } return deviceOpened, nil } + +func updatePVCStatusCapacity(pvcName string, pvc *v1.PersistentVolumeClaim, capacity v1.ResourceList, client clientset.Interface) error { + pvcCopy := pvc.DeepCopy() + + oldData, err := json.Marshal(pvcCopy) + if err != nil { + return fmt.Errorf("Failed to marshal oldData for pvc %q with %v", pvcName, err) + } + + pvcCopy.Status.Capacity = capacity + newData, err := json.Marshal(pvcCopy) + + if err != nil { + return fmt.Errorf("Failed to marshal newData for pvc %q with %v", pvcName, err) + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, pvcCopy) + + if err != nil { + return fmt.Errorf("Failed to CreateTwoWayMergePatch for pvc %q with %v ", pvcName, err) + } + _, err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Patch(pvcName, types.StrategicMergePatchType, patchBytes, "status") + if err != nil { + return fmt.Errorf("Failed to patch PVC %q with %v", pvcName, err) + } + return nil +} From 7be94c4b0646ab5a76f0dd0686f37e3da1f65e59 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 20 Nov 2017 13:02:13 -0500 Subject: [PATCH 3/4] Implement resizing support for GCE Fix GCE attacher test Update bazel files --- pkg/cloudprovider/providers/gce/BUILD | 1 + pkg/cloudprovider/providers/gce/gce_disks.go | 86 +++++++++++++++++++ .../providers/gce/gce_disks_test.go | 13 +++ pkg/volume/gce_pd/BUILD | 1 + pkg/volume/gce_pd/attacher_test.go | 8 ++ pkg/volume/gce_pd/gce_pd.go | 23 +++++ .../operationexecutor/operation_generator.go | 9 +- .../persistentvolume/resize/admission.go | 4 + 8 files changed, 141 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index bcd8c5de9dc..9636a85c9e3 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -67,6 +67,7 @@ go_library( "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/cloudprovider/providers/gce/gce_disks.go b/pkg/cloudprovider/providers/gce/gce_disks.go index 1e6615062be..6918c57c0df 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks.go +++ b/pkg/cloudprovider/providers/gce/gce_disks.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/cloudprovider" @@ -90,6 +91,9 @@ type diskServiceManager interface { instanceName string, devicePath string) (gceObject, error) + ResizeDiskOnCloudProvider(disk *GCEDisk, sizeGb int64, zone string) (gceObject, error) + RegionalResizeDiskOnCloudProvider(disk *GCEDisk, sizeGb int64) (gceObject, error) + // Gets the persistent disk from GCE with the given diskName. GetDiskFromCloudProvider(zone string, diskName string) (*GCEDisk, error) @@ -264,6 +268,7 @@ func (manager *gceServiceManager) GetDiskFromCloudProvider( Name: diskAlpha.Name, Kind: diskAlpha.Kind, Type: diskAlpha.Type, + SizeGb: diskAlpha.SizeGb, }, nil } @@ -289,6 +294,7 @@ func (manager *gceServiceManager) GetDiskFromCloudProvider( Name: diskStable.Name, Kind: diskStable.Kind, Type: diskStable.Type, + SizeGb: diskStable.SizeGb, }, nil } @@ -313,6 +319,7 @@ func (manager *gceServiceManager) GetRegionalDiskFromCloudProvider( Name: diskAlpha.Name, Kind: diskAlpha.Kind, Type: diskAlpha.Type, + SizeGb: diskAlpha.SizeGb, }, nil } @@ -469,6 +476,30 @@ func (manager *gceServiceManager) getRegionFromZone(zoneInfo zoneType) (string, return region, nil } +func (manager *gceServiceManager) ResizeDiskOnCloudProvider(disk *GCEDisk, sizeGb int64, zone string) (gceObject, error) { + if manager.gce.AlphaFeatureGate.Enabled(AlphaFeatureGCEDisk) { + resizeServiceRequest := &computealpha.DisksResizeRequest{ + SizeGb: sizeGb, + } + return manager.gce.serviceAlpha.Disks.Resize(manager.gce.projectID, zone, disk.Name, resizeServiceRequest).Do() + + } + resizeServiceRequest := &compute.DisksResizeRequest{ + SizeGb: sizeGb, + } + return manager.gce.service.Disks.Resize(manager.gce.projectID, zone, disk.Name, resizeServiceRequest).Do() +} + +func (manager *gceServiceManager) RegionalResizeDiskOnCloudProvider(disk *GCEDisk, sizeGb int64) (gceObject, error) { + if manager.gce.AlphaFeatureGate.Enabled(AlphaFeatureGCEDisk) { + resizeServiceRequest := &computealpha.RegionDisksResizeRequest{ + SizeGb: sizeGb, + } + return manager.gce.serviceAlpha.RegionDisks.Resize(manager.gce.projectID, disk.Region, disk.Name, resizeServiceRequest).Do() + } + return nil, fmt.Errorf("RegionalResizeDiskOnCloudProvider is a regional PD feature and is only available via the GCE Alpha API. Enable \"GCEDiskAlphaAPI\" in the list of \"alpha-features\" in \"gce.conf\" to use the feature.") +} + // Disks is interface for manipulation with GCE PDs. type Disks interface { // AttachDisk attaches given disk to the node with the specified NodeName. @@ -498,6 +529,9 @@ type Disks interface { // DeleteDisk deletes PD. DeleteDisk(diskToDelete string) error + // ResizeDisk resizes PD and returns new disk size + ResizeDisk(diskToResize string, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) + // GetAutoLabelsForPD returns labels to apply to PersistentVolume // representing this PD, namely failure domain and zone. // zone can be provided to specify the zone for the PD, @@ -517,6 +551,7 @@ type GCEDisk struct { Name string Kind string Type string + SizeGb int64 } type zoneType interface { @@ -801,6 +836,57 @@ func (gce *GCECloud) DeleteDisk(diskToDelete string) error { return err } +// ResizeDisk expands given disk and returns new disk size +func (gce *GCECloud) ResizeDisk(diskToResize string, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) { + disk, err := gce.GetDiskByNameUnknownZone(diskToResize) + if err != nil { + return oldSize, err + } + + requestBytes := newSize.Value() + // GCE resizes in chunks of GBs (not GiB) + requestGB := volume.RoundUpSize(requestBytes, 1000*1000*1000) + newSizeQuant := resource.MustParse(fmt.Sprintf("%dG", requestGB)) + + // If disk is already of size equal or greater than requested size, we simply return + if disk.SizeGb >= requestGB { + return newSizeQuant, nil + } + + var mc *metricContext + + switch zoneInfo := disk.ZoneInfo.(type) { + case singleZone: + mc = newDiskMetricContextZonal("resize", disk.Region, zoneInfo.zone) + resizeOp, err := gce.manager.ResizeDiskOnCloudProvider(disk, requestGB, zoneInfo.zone) + + if err != nil { + return oldSize, mc.Observe(err) + } + waitErr := gce.manager.WaitForZoneOp(resizeOp, zoneInfo.zone, mc) + if waitErr != nil { + return oldSize, waitErr + } + return newSizeQuant, nil + case multiZone: + mc = newDiskMetricContextRegional("resize", disk.Region) + resizeOp, err := gce.manager.RegionalResizeDiskOnCloudProvider(disk, requestGB) + + if err != nil { + return oldSize, mc.Observe(err) + } + waitErr := gce.manager.WaitForRegionalOp(resizeOp, mc) + if waitErr != nil { + return oldSize, waitErr + } + return newSizeQuant, nil + case nil: + return oldSize, fmt.Errorf("PD has nil ZoneInfo: %v", disk) + default: + return oldSize, fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo) + } +} + // Builds the labels that should be automatically added to a PersistentVolume backed by a GCE PD // Specifically, this builds FailureDomain (zone) and Region labels. // The PersistentVolumeLabel admission controller calls this and adds the labels when a PV is created. diff --git a/pkg/cloudprovider/providers/gce/gce_disks_test.go b/pkg/cloudprovider/providers/gce/gce_disks_test.go index 3c6b5ee4f70..34418d737a0 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks_test.go +++ b/pkg/cloudprovider/providers/gce/gce_disks_test.go @@ -897,6 +897,19 @@ func (manager *FakeServiceManager) GetRegionalDiskFromCloudProvider( }, nil } +func (manager *FakeServiceManager) ResizeDiskOnCloudProvider( + disk *GCEDisk, + size int64, + zone string) (gceObject, error) { + panic("Not implmented") +} + +func (manager *FakeServiceManager) RegionalResizeDiskOnCloudProvider( + disk *GCEDisk, + size int64) (gceObject, error) { + panic("Not implemented") +} + /** * Disk info is removed from the FakeServiceManager. */ diff --git a/pkg/volume/gce_pd/BUILD b/pkg/volume/gce_pd/BUILD index 45b973c02a1..8c83b4d0d70 100644 --- a/pkg/volume/gce_pd/BUILD +++ b/pkg/volume/gce_pd/BUILD @@ -47,6 +47,7 @@ go_test( "//pkg/volume/testing:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/volume/gce_pd/attacher_test.go b/pkg/volume/gce_pd/attacher_test.go index 3d8fe615a21..0539e11a572 100644 --- a/pkg/volume/gce_pd/attacher_test.go +++ b/pkg/volume/gce_pd/attacher_test.go @@ -22,6 +22,7 @@ import ( "testing" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" @@ -373,3 +374,10 @@ func (testcase *testcase) DeleteDisk(diskToDelete string) error { func (testcase *testcase) GetAutoLabelsForPD(name string, zone string) (map[string]string, error) { return map[string]string{}, errors.New("Not implemented") } + +func (testcase *testcase) ResizeDisk( + diskName string, + oldSize resource.Quantity, + newSize resource.Quantity) (resource.Quantity, error) { + return oldSize, errors.New("Not implemented") +} diff --git a/pkg/volume/gce_pd/gce_pd.go b/pkg/volume/gce_pd/gce_pd.go index 9ddeb95787e..5576f1fabea 100644 --- a/pkg/volume/gce_pd/gce_pd.go +++ b/pkg/volume/gce_pd/gce_pd.go @@ -47,6 +47,7 @@ var _ volume.VolumePlugin = &gcePersistentDiskPlugin{} var _ volume.PersistentVolumePlugin = &gcePersistentDiskPlugin{} var _ volume.DeletableVolumePlugin = &gcePersistentDiskPlugin{} var _ volume.ProvisionableVolumePlugin = &gcePersistentDiskPlugin{} +var _ volume.ExpandableVolumePlugin = &gcePersistentDiskPlugin{} const ( gcePersistentDiskPluginName = "kubernetes.io/gce-pd" @@ -190,6 +191,28 @@ func (plugin *gcePersistentDiskPlugin) newProvisionerInternal(options volume.Vol }, nil } +func (plugin *gcePersistentDiskPlugin) RequiresFSResize() bool { + return true +} + +func (plugin *gcePersistentDiskPlugin) ExpandVolumeDevice( + spec *volume.Spec, + newSize resource.Quantity, + oldSize resource.Quantity) (resource.Quantity, error) { + cloud, err := getCloudProvider(plugin.host.GetCloudProvider()) + + if err != nil { + return oldSize, err + } + pdName := spec.PersistentVolume.Spec.GCEPersistentDisk.PDName + updatedQuantity, err := cloud.ResizeDisk(pdName, oldSize, newSize) + + if err != nil { + return oldSize, err + } + return updatedQuantity, nil +} + func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index c99060ceafe..fc682fd607d 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -454,8 +454,9 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) - mounter := og.volumePluginMgr.Host.GetMounter(volumePlugin.GetPluginName()) - resizeError := og.resizeFileSystem(volumeToMount, devicePath, mounter) + // resizeFileSystem will resize the file system if user has requested a resize of + // underlying persistent volume and is allowed to do so. + resizeError := og.resizeFileSystem(volumeToMount, devicePath, volumePlugin.GetPluginName()) if resizeError != nil { return volumeToMount.GenerateErrorDetailed("MountVolume.Resize failed", resizeError) @@ -538,12 +539,12 @@ func (og *operationGenerator) GenerateMountVolumeFunc( }, volumePlugin.GetPluginName(), nil } -func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, mounter mount.Interface) error { +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, pluginName string) error { if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) return nil } - + mounter := og.volumePluginMgr.Host.GetMounter(pluginName) // Get expander, if possible expandableVolumePlugin, _ := og.volumePluginMgr.FindExpandablePluginBySpec(volumeToMount.VolumeSpec) diff --git a/plugin/pkg/admission/persistentvolume/resize/admission.go b/plugin/pkg/admission/persistentvolume/resize/admission.go index f5327ecf636..b550edb5ff4 100644 --- a/plugin/pkg/admission/persistentvolume/resize/admission.go +++ b/plugin/pkg/admission/persistentvolume/resize/admission.go @@ -155,6 +155,10 @@ func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVol if pv.Spec.Cinder != nil { return true } + + if pv.Spec.GCEPersistentDisk != nil { + return true + } return false } From 5ee4d2bbf5b61d5d8fc71b57f68955effc96e2d7 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 20 Nov 2017 13:10:04 -0500 Subject: [PATCH 4/4] Enable PersistentVolumeClaimResize admission plugin in default cluster settings The plugin itself will not do anything if resizing is not enabled. Move Resize plugin to start --- cluster/centos/config-default.sh | 2 +- cluster/gce/config-default.sh | 2 +- cluster/libvirt-coreos/util.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh index dff6b5f387c..90d13bee12b 100755 --- a/cluster/centos/config-default.sh +++ b/cluster/centos/config-default.sh @@ -120,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} # Admission Controllers to invoke prior to persisting objects in cluster # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,Priority,ResourceQuota"} +export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,ResourceQuota"} # Extra options to set on the Docker command line. # This is useful for setting --insecure-registry for local registries. diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 6d592aa9762..8bf206066be 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -284,7 +284,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then fi # Admission Controllers to invoke prior to persisting objects in cluster -ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority +ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy" diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index d63a247902a..9fe2170ea4d 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -27,7 +27,7 @@ source "$KUBE_ROOT/cluster/common.sh" export LIBVIRT_DEFAULT_URI=qemu:///system export SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true} -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota} +export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,ResourceQuota} readonly POOL=kubernetes readonly POOL_PATH=/var/lib/libvirt/images/kubernetes