diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index 59c5e9c5ba1..12740573811 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -52,6 +52,8 @@ const ( FailedDetachVolume = "FailedDetachVolume" FailedMountVolume = "FailedMount" VolumeResizeFailed = "VolumeResizeFailed" + FileSystemResizeFailed = "FileSystemResizeFailed" + FileSystemResizeSuccess = "FileSystemResizeSuccessful" FailedUnMountVolume = "FailedUnMount" FailedMapVolume = "FailedMapVolume" FailedUnmapDevice = "FailedUnmapDevice" diff --git a/pkg/util/BUILD b/pkg/util/BUILD index ae800c2ce19..362af715259 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -47,6 +47,7 @@ filegroup( "//pkg/util/procfs:all-srcs", "//pkg/util/reflector/prometheus:all-srcs", "//pkg/util/removeall:all-srcs", + "//pkg/util/resizefs:all-srcs", "//pkg/util/resourcecontainer:all-srcs", "//pkg/util/rlimit:all-srcs", "//pkg/util/selinux:all-srcs", diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 9caec6707b6..71064f321e3 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -498,7 +498,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, if mountErr != nil { // Mount failed. This indicates either that the disk is unformatted or // it contains an unexpected filesystem. - existingFormat, err := mounter.getDiskFormat(source) + existingFormat, err := mounter.GetDiskFormat(source) if err != nil { return err } @@ -536,8 +536,8 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, return mountErr } -// getDiskFormat uses 'lsblk' to see if the given disk is unformated -func (mounter *SafeFormatAndMount) getDiskFormat(disk string) (string, error) { +// GetDiskFormat uses 'lsblk' to see if the given disk is unformated +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { args := []string{"-n", "-o", "FSTYPE", disk} glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) dataOut, err := mounter.Exec.Run("lsblk", args...) diff --git a/pkg/util/resizefs/BUILD b/pkg/util/resizefs/BUILD new file mode 100644 index 00000000000..aaa1f9aa5f4 --- /dev/null +++ b/pkg/util/resizefs/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "resizefs_unsupported.go", + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "resizefs_linux.go", + ], + "//conditions:default": [], + }), + importpath = "k8s.io/kubernetes/pkg/util/resizefs", + visibility = ["//visibility:public"], + deps = [ + "//pkg/util/mount:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/util/resizefs/resizefs_linux.go b/pkg/util/resizefs/resizefs_linux.go new file mode 100644 index 00000000000..6a4d82d6c03 --- /dev/null +++ b/pkg/util/resizefs/resizefs_linux.go @@ -0,0 +1,143 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resizefs + +import ( + "fmt" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/mount" + utilexec "k8s.io/utils/exec" +) + +const ( + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 +) + +// ResizeFs Provides support for resizing file systems +type ResizeFs struct { + mounter *mount.SafeFormatAndMount +} + +// NewResizeFs returns new instance of resizer +func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { + return &ResizeFs{mounter: mounter} +} + +// Resize perform resize of file system +func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { + format, err := resizefs.mounter.GetDiskFormat(devicePath) + + if err != nil { + formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err) + return false, formatErr + } + + // If disk has no format, there is no need to resize the disk because mkfs.* + // by default will use whole disk anyways. + if format == "" { + return false, nil + } + + deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath) + + if err != nil { + deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err) + return false, deviceOpenErr + } + + if deviceOpened { + deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath) + return false, deviceAlreadyOpenErr + } + + switch format { + case "ext3", "ext4": + fsckErr := resizefs.extFsck(devicePath, format) + if fsckErr != nil { + return false, fsckErr + } + return resizefs.extResize(devicePath) + case "xfs": + fsckErr := resizefs.fsckDevice(devicePath) + if fsckErr != nil { + return false, fsckErr + } + return resizefs.xfsResize(devicePath) + } + return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath) +} + +func (resizefs *ResizeFs) fsckDevice(devicePath string) error { + glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath) + args := []string{"-a", devicePath} + out, err := resizefs.mounter.Exec.Run("fsck", args...) + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out)) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + glog.Infof("`fsck` error %s", string(out)) + } + } + return nil +} + +func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error { + glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath) + args := []string{"-f", "-y", devicePath} + out, err := resizefs.mounter.Run("fsck."+fsType, args...) + if err != nil { + return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out)) + } + return nil +} + +func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { + output, err := resizefs.mounter.Exec.Run("resize2fs", devicePath) + if err == nil { + glog.V(2).Infof("Device %s resized successfully", devicePath) + return true, nil + } + + resizeError := fmt.Errorf("resize of device %s failed: %v. resize2fs output: %s", devicePath, err, string(output)) + return false, resizeError + +} + +func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) { + args := []string{"-d", devicePath} + output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...) + + if err == nil { + glog.V(2).Infof("Device %s resized successfully", devicePath) + return true, nil + } + + resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output)) + return false, resizeError +} diff --git a/pkg/util/resizefs/resizefs_unsupported.go b/pkg/util/resizefs/resizefs_unsupported.go new file mode 100644 index 00000000000..9241d7d5b27 --- /dev/null +++ b/pkg/util/resizefs/resizefs_unsupported.go @@ -0,0 +1,40 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resizefs + +import ( + "fmt" + + "k8s.io/kubernetes/pkg/util/mount" +) + +// ResizeFs Provides support for resizing file systems +type ResizeFs struct { + mounter *mount.SafeFormatAndMount +} + +// NewResizeFs returns new instance of resizer +func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { + return &ResizeFs{mounter: mounter} +} + +// Resize perform resize of file system +func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { + return false, fmt.Errorf("Resize is not supported for this build") +} diff --git a/pkg/volume/util/operationexecutor/BUILD b/pkg/volume/util/operationexecutor/BUILD index 36eafd136d1..15c2ac27d83 100644 --- a/pkg/volume/util/operationexecutor/BUILD +++ b/pkg/volume/util/operationexecutor/BUILD @@ -18,6 +18,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/util/mount:go_default_library", + "//pkg/util/resizefs:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/nestedpendingoperations:go_default_library", @@ -28,6 +29,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", diff --git a/pkg/volume/util/operationexecutor/operation_executor_test.go b/pkg/volume/util/operationexecutor/operation_executor_test.go index 3624c889156..4e06b39616b 100644 --- a/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -80,7 +80,6 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForAttachablePlugins(t *te volumesToMount := make([]VolumeToMount, numVolumesToAttach) pdName := "pd-volume" volumeName := v1.UniqueVolumeName(pdName) - // Act for i := range volumesToMount { podName := "pod-" + strconv.Itoa((i + 1)) diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 909b3aae745..c99060ceafe 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -17,6 +17,7 @@ limitations under the License. package operationexecutor import ( + "encoding/json" "fmt" "strings" "time" @@ -26,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" @@ -33,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/features" kevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/util/resizefs" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" @@ -451,6 +454,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + mounter := og.volumePluginMgr.Host.GetMounter(volumePlugin.GetPluginName()) + resizeError := og.resizeFileSystem(volumeToMount, devicePath, mounter) + + if resizeError != nil { + return volumeToMount.GenerateErrorDetailed("MountVolume.Resize failed", resizeError) + } + deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { @@ -528,6 +538,65 @@ func (og *operationGenerator) GenerateMountVolumeFunc( }, volumePlugin.GetPluginName(), nil } +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, mounter mount.Interface) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) + return nil + } + + // Get expander, if possible + expandableVolumePlugin, _ := + og.volumePluginMgr.FindExpandablePluginBySpec(volumeToMount.VolumeSpec) + + if expandableVolumePlugin != nil && + expandableVolumePlugin.RequiresFSResize() && + volumeToMount.VolumeSpec.PersistentVolume != nil { + pv := volumeToMount.VolumeSpec.PersistentVolume + pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(pv.Spec.ClaimRef.Name, metav1.GetOptions{}) + if err != nil { + // Return error rather than leave the file system un-resized, caller will log and retry + return volumeToMount.GenerateErrorDetailed("MountVolume get PVC failed", err) + } + + pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage] + pvSpecCap := pv.Spec.Capacity[v1.ResourceStorage] + if pvcStatusCap.Cmp(pvSpecCap) < 0 { + // File system resize was requested, proceed + glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) + + diskFormatter := &mount.SafeFormatAndMount{ + Interface: mounter, + Exec: og.volumePluginMgr.Host.GetExec(expandableVolumePlugin.GetPluginName()), + } + + resizer := resizefs.NewResizeFs(diskFormatter) + resizeStatus, resizeErr := resizer.Resize(devicePath) + + if resizeErr != nil { + resizeDetailedError := volumeToMount.GenerateErrorDetailed("MountVolume.resizeFileSystem failed", resizeErr) + glog.Error(resizeDetailedError) + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, resizeDetailedError.Error()) + return resizeDetailedError + } + + if resizeStatus { + simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem succeeded", "") + og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg) + glog.Infof(detailedMsg) + } + + // File system resize succeeded, now update the PVC's Capacity to match the PV's + err = updatePVCStatusCapacity(pvc.Name, pvc, pv.Spec.Capacity, og.kubeClient) + if err != nil { + // On retry, resizeFileSystem will be called again but do nothing + return volumeToMount.GenerateErrorDetailed("MountVolume update PVC status failed", err) + } + return nil + } + } + return nil +} + func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { @@ -1104,6 +1173,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, expandErr.Error()) return expandErr } + glog.Infof("ExpandVolume succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) newSize = updatedSize // k8s doesn't have transactions, we can't guarantee that after updating PV - updating PVC will be // successful, that is why all PVCs for which pvc.Spec.Size > pvc.Status.Size must be reprocessed @@ -1115,6 +1185,7 @@ func (og *operationGenerator) GenerateExpandVolumeFunc( og.recorder.Eventf(pvcWithResizeRequest.PVC, v1.EventTypeWarning, kevents.VolumeResizeFailed, updateErr.Error()) return updateErr } + glog.Infof("ExpandVolume.UpdatePV succeeded for volume %s", pvcWithResizeRequest.QualifiedName()) } // No Cloudprovider resize needed, lets mark resizing as done @@ -1190,3 +1261,30 @@ func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (boo } return deviceOpened, nil } + +func updatePVCStatusCapacity(pvcName string, pvc *v1.PersistentVolumeClaim, capacity v1.ResourceList, client clientset.Interface) error { + pvcCopy := pvc.DeepCopy() + + oldData, err := json.Marshal(pvcCopy) + if err != nil { + return fmt.Errorf("Failed to marshal oldData for pvc %q with %v", pvcName, err) + } + + pvcCopy.Status.Capacity = capacity + newData, err := json.Marshal(pvcCopy) + + if err != nil { + return fmt.Errorf("Failed to marshal newData for pvc %q with %v", pvcName, err) + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, pvcCopy) + + if err != nil { + return fmt.Errorf("Failed to CreateTwoWayMergePatch for pvc %q with %v ", pvcName, err) + } + _, err = client.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Patch(pvcName, types.StrategicMergePatchType, patchBytes, "status") + if err != nil { + return fmt.Errorf("Failed to patch PVC %q with %v", pvcName, err) + } + return nil +}