Merge pull request #101882 from jackfrancis/kubelet-initialnode-getcapacity

kubelet: more resilient node allocatable ephemeral-storage data getter

pkg/kubelet/cm/container_manager_linux.go

@@ -914,7 +914,31 @@ func isKernelPid(pid int) bool {
 	return err != nil && os.IsNotExist(err)
 }
 
+// GetCapacity returns node capacity data for "cpu", "memory", "ephemeral-storage", and "huge-pages*"
+// At present this method is only invoked when introspecting ephemeral storage
 func (cm *containerManagerImpl) GetCapacity() v1.ResourceList {
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) {
+		// We store allocatable ephemeral-storage in the capacity property once we Start() the container manager
+		if _, ok := cm.capacity[v1.ResourceEphemeralStorage]; !ok {
+			// If we haven't yet stored the capacity for ephemeral-storage, we can try to fetch it directly from cAdvisor,
+			if cm.cadvisorInterface != nil {
+				rootfs, err := cm.cadvisorInterface.RootFsInfo()
+				if err != nil {
+					klog.ErrorS(err, "Unable to get rootfs data from cAdvisor interface")
+					// If the rootfsinfo retrieval from cAdvisor fails for any reason, fallback to returning the capacity property with no ephemeral storage data
+					return cm.capacity
+				}
+				// We don't want to mutate cm.capacity here so we'll manually construct a v1.ResourceList from it,
+				// and add ephemeral-storage
+				capacityWithEphemeralStorage := v1.ResourceList{}
+				for rName, rQuant := range cm.capacity {
+					capacityWithEphemeralStorage[rName] = rQuant
+				}
+				capacityWithEphemeralStorage[v1.ResourceEphemeralStorage] = cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs)[v1.ResourceEphemeralStorage]
+				return capacityWithEphemeralStorage
+			}
+		}
+	}
 	return cm.capacity
 }
 

pkg/kubelet/cm/container_manager_linux_test.go

@@ -20,14 +20,24 @@ limitations under the License.
 package cm
 
 import (
+	"errors"
 	"io/ioutil"
 	"os"
 	"path"
 	"testing"
 
+	gomock "github.com/golang/mock/gomock"
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
+
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 
 	"k8s.io/mount-utils"
 )
@@ -166,3 +176,95 @@ func TestSoftRequirementsValidationSuccess(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, f.cpuHardcapping, "cpu hardcapping is expected to be enabled")
 }
+
+func TestGetCapacity(t *testing.T) {
+	ephemeralStorageFromCapacity := int64(2000)
+	ephemeralStorageFromCadvisor := int64(8000)
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	mockCtrlError := gomock.NewController(t)
+	defer mockCtrlError.Finish()
+
+	mockCadvisor := cadvisortest.NewMockInterface(mockCtrl)
+	rootfs := cadvisorapiv2.FsInfo{
+		Capacity: 8000,
+	}
+	mockCadvisor.EXPECT().RootFsInfo().Return(rootfs, nil)
+	mockCadvisorError := cadvisortest.NewMockInterface(mockCtrlError)
+	mockCadvisorError.EXPECT().RootFsInfo().Return(cadvisorapiv2.FsInfo{}, errors.New("Unable to get rootfs data from cAdvisor interface"))
+	cases := []struct {
+		name                                string
+		cm                                  *containerManagerImpl
+		expectedResourceQuantity            *resource.Quantity
+		expectedNoEphemeralStorage          bool
+		enableLocalStorageCapacityIsolation bool
+	}{
+		{
+			name: "capacity property has ephemeral-storage",
+			cm: &containerManagerImpl{
+				cadvisorInterface: mockCadvisor,
+				capacity: v1.ResourceList{
+					v1.ResourceEphemeralStorage: *resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
+				},
+			},
+			expectedResourceQuantity:            resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
+			expectedNoEphemeralStorage:          false,
+			enableLocalStorageCapacityIsolation: true,
+		},
+		{
+			name: "capacity property does not have ephemeral-storage",
+			cm: &containerManagerImpl{
+				cadvisorInterface: mockCadvisor,
+				capacity:          v1.ResourceList{},
+			},
+			expectedResourceQuantity:            resource.NewQuantity(ephemeralStorageFromCadvisor, resource.BinarySI),
+			expectedNoEphemeralStorage:          false,
+			enableLocalStorageCapacityIsolation: true,
+		},
+		{
+			name: "capacity property does not have ephemeral-storage, error from rootfs",
+			cm: &containerManagerImpl{
+				cadvisorInterface: mockCadvisorError,
+				capacity:          v1.ResourceList{},
+			},
+			expectedNoEphemeralStorage:          true,
+			enableLocalStorageCapacityIsolation: true,
+		},
+		{
+			name: "capacity property does not have ephemeral-storage, cadvisor interface is nil",
+			cm: &containerManagerImpl{
+				cadvisorInterface: nil,
+				capacity:          v1.ResourceList{},
+			},
+			expectedNoEphemeralStorage:          true,
+			enableLocalStorageCapacityIsolation: true,
+		},
+		{
+			name: "LocalStorageCapacityIsolation feature flag is disabled",
+			cm: &containerManagerImpl{
+				cadvisorInterface: mockCadvisor,
+				capacity: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("4"),
+					v1.ResourceMemory: resource.MustParse("16G"),
+				},
+			},
+			expectedNoEphemeralStorage:          true,
+			enableLocalStorageCapacityIsolation: false,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.LocalStorageCapacityIsolation, c.enableLocalStorageCapacityIsolation)()
+			ret := c.cm.GetCapacity()
+			if v, exists := ret[v1.ResourceEphemeralStorage]; !exists {
+				if !c.expectedNoEphemeralStorage {
+					t.Errorf("did not get any ephemeral storage data")
+				}
+			} else {
+				if v.Value() != c.expectedResourceQuantity.Value() {
+					t.Errorf("got unexpected %s value, expected %d, got %d", v1.ResourceEphemeralStorage, c.expectedResourceQuantity.Value(), v.Value())
+				}
+			}
+		})
+	}
+}