diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 64a284195d0..b86106a5261 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -92,6 +92,7 @@ type KubeletServer struct { StreamingConnectionIdleTimeout time.Duration ImageGCHighThresholdPercent int ImageGCLowThresholdPercent int + LowDiskSpaceThresholdMB int NetworkPluginName string CloudProvider string CloudConfigFile string @@ -152,6 +153,7 @@ func NewKubeletServer() *KubeletServer { MasterServiceNamespace: api.NamespaceDefault, ImageGCHighThresholdPercent: 90, ImageGCLowThresholdPercent: 80, + LowDiskSpaceThresholdMB: 256, NetworkPluginName: "", HostNetworkSources: kubelet.FileSource, CertDirectory: "/var/run/kubernetes", @@ -208,6 +210,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.NodeStatusUpdateFrequency, "node-status-update-frequency", s.NodeStatusUpdateFrequency, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s") fs.IntVar(&s.ImageGCHighThresholdPercent, "image-gc-high-threshold", s.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Default: 90%%") fs.IntVar(&s.ImageGCLowThresholdPercent, "image-gc-low-threshold", s.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. Default: 80%%") + fs.IntVar(&s.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", s.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected. Default: 256") fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") @@ -251,6 +254,10 @@ func (s *KubeletServer) Run(_ []string) error { LowThresholdPercent: s.ImageGCLowThresholdPercent, } + diskSpacePolicy := kubelet.DiskSpacePolicy{ + DockerFreeDiskMB: s.LowDiskSpaceThresholdMB, + RootFreeDiskMB: s.LowDiskSpaceThresholdMB, + } cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) glog.Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) @@ -318,6 +325,7 @@ func (s *KubeletServer) Run(_ []string) error { StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout, TLSOptions: tlsOptions, ImageGCPolicy: imageGCPolicy, + DiskSpacePolicy: diskSpacePolicy, Cloud: cloud, NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, ResourceContainer: s.ResourceContainer, @@ -406,6 +414,10 @@ func SimpleKubelet(client *client.Client, HighThresholdPercent: 90, LowThresholdPercent: 80, } + diskSpacePolicy := kubelet.DiskSpacePolicy{ + DockerFreeDiskMB: 256, + RootFreeDiskMB: 256, + } kcfg := KubeletConfig{ KubeClient: client, DockerClient: dockerClient, @@ -429,6 +441,7 @@ func SimpleKubelet(client *client.Client, CadvisorInterface: cadvisorInterface, ConfigFile: configFilePath, ImageGCPolicy: imageGCPolicy, + DiskSpacePolicy: diskSpacePolicy, Cloud: cloud, NodeStatusUpdateFrequency: 10 * time.Second, ResourceContainer: "/kubelet", @@ -560,6 +573,7 @@ type KubeletConfig struct { Recorder record.EventRecorder TLSOptions *kubelet.TLSOptions ImageGCPolicy kubelet.ImageGCPolicy + DiskSpacePolicy kubelet.DiskSpacePolicy Cloud cloudprovider.Interface NodeStatusUpdateFrequency time.Duration ResourceContainer string @@ -609,6 +623,7 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.Recorder, kc.CadvisorInterface, kc.ImageGCPolicy, + kc.DiskSpacePolicy, kc.Cloud, kc.NodeStatusUpdateFrequency, kc.ResourceContainer, diff --git a/pkg/kubelet/cadvisor/cadvisor_fake.go b/pkg/kubelet/cadvisor/cadvisor_fake.go index 01858febd2f..18683ceb0d8 100644 --- a/pkg/kubelet/cadvisor/cadvisor_fake.go +++ b/pkg/kubelet/cadvisor/cadvisor_fake.go @@ -52,6 +52,10 @@ func (c *Fake) DockerImagesFsInfo() (cadvisorApiV2.FsInfo, error) { return cadvisorApiV2.FsInfo{}, nil } +func (c *Fake) RootFsInfo() (cadvisorApiV2.FsInfo, error) { + return cadvisorApiV2.FsInfo{}, nil +} + func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error) { return new(events.EventChannel), nil } diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index c56016b93a7..339c0340209 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -132,16 +132,24 @@ func (cc *cadvisorClient) MachineInfo() (*cadvisorApi.MachineInfo, error) { } func (cc *cadvisorClient) DockerImagesFsInfo() (cadvisorApiV2.FsInfo, error) { - res, err := cc.GetFsInfo(cadvisorFs.LabelDockerImages) + return cc.getFsInfo(cadvisorFs.LabelDockerImages) +} + +func (cc *cadvisorClient) RootFsInfo() (cadvisorApiV2.FsInfo, error) { + return cc.getFsInfo(cadvisorFs.LabelSystemRoot) +} + +func (cc *cadvisorClient) getFsInfo(label string) (cadvisorApiV2.FsInfo, error) { + res, err := cc.GetFsInfo(label) if err != nil { return cadvisorApiV2.FsInfo{}, err } if len(res) == 0 { - return cadvisorApiV2.FsInfo{}, fmt.Errorf("failed to find information for the filesystem containing Docker images") + return cadvisorApiV2.FsInfo{}, fmt.Errorf("failed to find information for the filesystem labeled %q", label) } - // TODO(vmarmol): Handle this better when Docker has more than one image filesystem. + // TODO(vmarmol): Handle this better when a label has more than one image filesystem. if len(res) > 1 { - glog.Warningf("More than one Docker images filesystem: %#v. Only using the first one", res) + glog.Warningf("More than one filesystem labeled %q: %#v. Only using the first one", label, res) } return res[0], nil diff --git a/pkg/kubelet/cadvisor/cadvisor_mock.go b/pkg/kubelet/cadvisor/cadvisor_mock.go index 096c3d37cf6..3e3a2c3e7d9 100644 --- a/pkg/kubelet/cadvisor/cadvisor_mock.go +++ b/pkg/kubelet/cadvisor/cadvisor_mock.go @@ -62,6 +62,11 @@ func (c *Mock) DockerImagesFsInfo() (cadvisorApiV2.FsInfo, error) { return args.Get(0).(cadvisorApiV2.FsInfo), args.Error(1) } +func (c *Mock) RootFsInfo() (cadvisorApiV2.FsInfo, error) { + args := c.Called() + return args.Get(0).(cadvisorApiV2.FsInfo), args.Error(1) +} + func (c *Mock) WatchEvents(request *events.Request) (*events.EventChannel, error) { args := c.Called() return args.Get(0).(*events.EventChannel), args.Error(1) diff --git a/pkg/kubelet/cadvisor/cadvisor_unsupported.go b/pkg/kubelet/cadvisor/cadvisor_unsupported.go index c4f021abca1..ca232346d58 100644 --- a/pkg/kubelet/cadvisor/cadvisor_unsupported.go +++ b/pkg/kubelet/cadvisor/cadvisor_unsupported.go @@ -61,6 +61,10 @@ func (cu *cadvisorUnsupported) DockerImagesFsInfo() (cadvisorApiV2.FsInfo, error return cadvisorApiV2.FsInfo{}, unsupportedErr } +func (cu *cadvisorUnsupported) RootFsInfo() (cadvisorApiV2.FsInfo, error) { + return cadvisorApiV2.FsInfo{}, unsupportedErr +} + func (cu *cadvisorUnsupported) WatchEvents(request *events.Request) (*events.EventChannel, error) { return nil, unsupportedErr } diff --git a/pkg/kubelet/cadvisor/types.go b/pkg/kubelet/cadvisor/types.go index f6ae05d3151..8b855b52a43 100644 --- a/pkg/kubelet/cadvisor/types.go +++ b/pkg/kubelet/cadvisor/types.go @@ -34,6 +34,9 @@ type Interface interface { // Returns usage information about the filesystem holding Docker images. DockerImagesFsInfo() (cadvisorApiV2.FsInfo, error) + // Returns usage information about the root filesystem. + RootFsInfo() (cadvisorApiV2.FsInfo, error) + // Get events streamed through passedChannel that fit the request. WatchEvents(request *events.Request) (*events.EventChannel, error) } diff --git a/pkg/kubelet/disk_manager.go b/pkg/kubelet/disk_manager.go new file mode 100644 index 00000000000..4f21f0d6252 --- /dev/null +++ b/pkg/kubelet/disk_manager.go @@ -0,0 +1,149 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "sync" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" + "github.com/golang/glog" + cadvisorApi "github.com/google/cadvisor/info/v2" +) + +// Manages policy for diskspace management for disks holding docker images and root fs. + +// Implementation is thread-safe. +type diskSpaceManager interface { + // Checks the available disk space + IsRootDiskSpaceAvailable() (bool, error) + IsDockerDiskSpaceAvailable() (bool, error) + // Always returns sufficient space till Unfreeze() is called. + // This is a signal from caller that its internal initialization is done. + Unfreeze() +} + +type DiskSpacePolicy struct { + // free disk space threshold for filesystem holding docker images. + DockerFreeDiskMB int + // free disk space threshold for root filesystem. Host volumes are created on root fs. + RootFreeDiskMB int +} + +type fsInfo struct { + Usage int64 + Capacity int64 + Timestamp time.Time +} + +type realDiskSpaceManager struct { + cadvisor cadvisor.Interface + cachedInfo map[string]fsInfo // cache of filesystem info. + lock sync.Mutex // protecting cachedInfo and frozen. + policy DiskSpacePolicy // thresholds. Set at creation time. + frozen bool // space checks always return ok when frozen is set. True on creation. +} + +func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorApi.FsInfo, error)) (fsInfo, error) { + dm.lock.Lock() + defer dm.lock.Unlock() + fsi := fsInfo{} + if info, ok := dm.cachedInfo[fsType]; ok { + timeLimit := time.Now().Add(-2 * time.Second) + if info.Timestamp.After(timeLimit) { + fsi = info + } + } + if fsi.Timestamp.IsZero() { + fs, err := f() + if err != nil { + return fsInfo{}, err + } + fsi.Timestamp = time.Now() + fsi.Usage = int64(fs.Usage) + fsi.Capacity = int64(fs.Capacity) + dm.cachedInfo[fsType] = fsi + } + return fsi, nil +} + +func (dm *realDiskSpaceManager) IsDockerDiskSpaceAvailable() (bool, error) { + return dm.isSpaceAvailable("docker", dm.policy.DockerFreeDiskMB, dm.cadvisor.DockerImagesFsInfo) +} + +func (dm *realDiskSpaceManager) IsRootDiskSpaceAvailable() (bool, error) { + return dm.isSpaceAvailable("root", dm.policy.RootFreeDiskMB, dm.cadvisor.RootFsInfo) +} + +func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f func() (cadvisorApi.FsInfo, error)) (bool, error) { + if dm.frozen { + return true, nil + } + fsInfo, err := dm.getFsInfo(fsType, f) + if err != nil { + return true, fmt.Errorf("failed to get fs info for %q: %v", fsType, err) + } + if fsInfo.Capacity == 0 { + return true, fmt.Errorf("could not determine capacity for %q fs.", fsType) + } + free := fsInfo.Capacity - fsInfo.Usage + if free < 0 { + return true, fmt.Errorf("wrong fs usage for %q: capacity %d, usage %d", fsType, fsInfo.Capacity, fsInfo.Usage) + } + + const mb = int64(1024 * 1024) + + if free < int64(threshold)*mb { + glog.Infof("Running out of space on disk for %q: free %d MB, threshold %d MB", fsType, free/mb, threshold) + return false, nil + } + return true, nil +} + +func (dm *realDiskSpaceManager) Unfreeze() { + dm.lock.Lock() + defer dm.lock.Unlock() + dm.frozen = false +} + +func validatePolicy(policy DiskSpacePolicy) error { + if policy.DockerFreeDiskMB < 0 { + return fmt.Errorf("free disk space should be non-negative. Invalid value %d for docker disk space threshold.", policy.DockerFreeDiskMB) + } + if policy.RootFreeDiskMB < 0 { + return fmt.Errorf("free disk space should be non-negative. Invalid value %d for root disk space threshold.", policy.RootFreeDiskMB) + } + return nil +} + +func newDiskSpaceManager(cadvisorInterface cadvisor.Interface, policy DiskSpacePolicy) (diskSpaceManager, error) { + // validate policy + err := validatePolicy(policy) + if err != nil { + return nil, err + } + + dm := &realDiskSpaceManager{ + cadvisor: cadvisorInterface, + policy: policy, + cachedInfo: map[string]fsInfo{}, + frozen: true, + } + + return dm, nil +} diff --git a/pkg/kubelet/disk_manager_test.go b/pkg/kubelet/disk_manager_test.go new file mode 100644 index 00000000000..b94db837c65 --- /dev/null +++ b/pkg/kubelet/disk_manager_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" + cadvisorApi "github.com/google/cadvisor/info/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testPolicy() DiskSpacePolicy { + return DiskSpacePolicy{ + DockerFreeDiskMB: 250, + RootFreeDiskMB: 250, + } +} + +func testValidPolicy(t *testing.T) { + assert := assert.New(t) + policy := testPolicy() + c := new(cadvisor.Mock) + _, err := newDiskSpaceManager(c, policy) + require.NoError(t, err) + + policy = testPolicy() + policy.DockerFreeDiskMB = -1 + _, err = newDiskSpaceManager(c, policy) + assert.Error(err) + + policy = testPolicy() + policy.RootFreeDiskMB = -1 + _, err = newDiskSpaceManager(c, policy) + assert.Error(err) +} + +func testSpaceAvailable(t *testing.T) { + policy := testPolicy() + mockCadvisor := new(cadvisor.Mock) + dm, err := newDiskSpaceManager(mockCadvisor, policy) + require.NoError(t, err) + const mb = 1024 * 1024 + mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 400 * mb, + Capacity: 1000 * mb, + }, nil) + mockCadvisor.On("RootFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 9 * mb, + Capacity: 10 * mb, + }, nil) + ok, err := dm.IsDockerDiskSpaceAvailable() + require.NoError(t, err) + require.True(t, ok) + ok, err = dm.IsRootDiskSpaceAvailable() + require.NoError(t, err) + require.True(t, ok) +} + +func testRootFsAvailable(t *testing.T) { + policy := testPolicy() + policy.RootFreeDiskMB = 10 + mockCadvisor := new(cadvisor.Mock) + dm, err := newDiskSpaceManager(mockCadvisor, policy) + require.NoError(t, err) + + const mb = 1024 * 1024 + // 500MB free + mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 9500 * mb, + Capacity: 10000 * mb, + }, nil) + // 10MB free + mockCadvisor.On("RootFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 990 * mb, + Capacity: 1000 * mb, + }, nil) + ok, err := dm.IsDockerDiskSpaceAvailable() + require.NoError(t, err) + require.True(t, ok) + ok, err = dm.IsRootDiskSpaceAvailable() + require.NoError(t, err) + require.False(t, ok) +} + +func testFsInfoError(t *testing.T) { + assert := assert.New(t) + policy := testPolicy() + policy.RootFreeDiskMB = 10 + mockCadvisor := new(cadvisor.Mock) + dm, err := newDiskSpaceManager(mockCadvisor, policy) + require.NoError(t, err) + + mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApi.FsInfo{}, fmt.Errorf("can't find fs")) + mockCadvisor.On("RootFsInfo").Return(cadvisorApi.FsInfo{}, fmt.Errorf("EBUSY")) + ok, err := dm.IsDockerDiskSpaceAvailable() + assert.Error(err) + require.True(t, ok) + ok, err = dm.IsRootDiskSpaceAvailable() + assert.Error(err) + require.True(t, ok) +} + +func testCache(t *testing.T) { + policy := testPolicy() + mockCadvisor := new(cadvisor.Mock) + dm, err := newDiskSpaceManager(mockCadvisor, policy) + require.NoError(t, err) + const mb = 1024 * 1024 + mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 400 * mb, + Capacity: 1000 * mb, + }, nil).Once() + mockCadvisor.On("RootFsInfo").Return(cadvisorApi.FsInfo{ + Usage: 9 * mb, + Capacity: 10 * mb, + }, nil).Once() + ok, err := dm.IsDockerDiskSpaceAvailable() + ok, err = dm.IsRootDiskSpaceAvailable() + + // Checking for space again shouldn't need to mock as cache would serve it. + ok, err = dm.IsDockerDiskSpaceAvailable() + require.NoError(t, err) + require.True(t, ok) + ok, err = dm.IsRootDiskSpaceAvailable() + require.NoError(t, err) + require.True(t, ok) +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index a0aaf59eff6..011f33bac63 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -130,6 +130,7 @@ func NewMainKubelet( recorder record.EventRecorder, cadvisorInterface cadvisor.Interface, imageGCPolicy ImageGCPolicy, + diskSpacePolicy DiskSpacePolicy, cloud cloudprovider.Interface, nodeStatusUpdateFrequency time.Duration, resourceContainer string, @@ -197,6 +198,10 @@ func NewMainKubelet( if err != nil { return nil, fmt.Errorf("failed to initialize image manager: %v", err) } + diskSpaceManager, err := newDiskSpaceManager(cadvisorInterface, diskSpacePolicy) + if err != nil { + return nil, fmt.Errorf("failed to initialize disk manager: %v", err) + } statusManager := newStatusManager(kubeClient) readinessManager := kubecontainer.NewReadinessManager() containerRefManager := kubecontainer.NewRefManager() @@ -228,6 +233,7 @@ func NewMainKubelet( cadvisor: cadvisorInterface, containerGC: containerGC, imageManager: imageManager, + diskSpaceManager: diskSpaceManager, statusManager: statusManager, volumeManager: volumeManager, cloud: cloud, @@ -399,6 +405,9 @@ type Kubelet struct { // Manager for images. imageManager imageManager + // Diskspace manager. + diskSpaceManager diskSpaceManager + // Cached MachineInfo returned by cadvisor. machineInfo *cadvisorApi.MachineInfo @@ -1136,6 +1145,9 @@ func (kl *Kubelet) SyncPods(allPods []*api.Pod, podSyncTypes map[types.UID]metri // Reject pods that we cannot run. kl.handleNotFittingPods(allPods) + // Reject new creation requests if diskspace is running low. + kl.handleOutOfDisk(allPods, podSyncTypes) + // Pod phase progresses monotonically. Once a pod has reached a final state, // it should never leave irregardless of the restart policy. The statuses // of such pods should not be changed, and there is no need to sync them. @@ -1306,6 +1318,44 @@ func (kl *Kubelet) checkCapacityExceeded(pods []*api.Pod) (fitting []*api.Pod, n return scheduler.CheckPodsExceedingCapacity(pods, capacity) } +// handleOutOfDisk detects if pods can't fit due to lack of disk space. +func (kl *Kubelet) handleOutOfDisk(pods []*api.Pod, podSyncTypes map[types.UID]metrics.SyncPodType) { + if len(podSyncTypes) == 0 { + // regular sync. no new pods + return + } + outOfDockerDisk := false + outOfRootDisk := false + // Check disk space once globally and reject or accept all new pods. + withinBounds, err := kl.diskSpaceManager.IsDockerDiskSpaceAvailable() + // Assume enough space in case of errors. + if err == nil && !withinBounds { + outOfDockerDisk = true + } + + withinBounds, err = kl.diskSpaceManager.IsRootDiskSpaceAvailable() + // Assume enough space in case of errors. + if err == nil && !withinBounds { + outOfRootDisk = true + } + // Kubelet would indicate all pods as newly created on the first run after restart. + // We ignore the first disk check to ensure that running pods are not killed. + // Disk manager will only declare out of disk problems if unfreeze has been called. + kl.diskSpaceManager.Unfreeze() + if outOfDockerDisk || outOfRootDisk { + for _, pod := range pods { + // Only reject pods that didn't start yet. + if podSyncTypes[pod.UID] == metrics.SyncPodCreate { + kl.recorder.Eventf(pod, "OutOfDisk", "Cannot start the pod due to lack of disk space.") + kl.statusManager.SetPodStatus(pod, api.PodStatus{ + Phase: api.PodFailed, + Message: "Pod cannot be started due to lack of disk space."}) + continue + } + } + } +} + // checkNodeSelectorMatching detects pods that do not match node's labels. func (kl *Kubelet) checkNodeSelectorMatching(pods []*api.Pod) (fitting []*api.Pod, notFitting []*api.Pod) { node, err := kl.GetNode() diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 0e48a6ddf13..e88e399be68 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -51,6 +51,7 @@ import ( _ "github.com/GoogleCloudPlatform/kubernetes/pkg/volume/host_path" "github.com/fsouza/go-dockerclient" cadvisorApi "github.com/google/cadvisor/info/v1" + cadvisorApiv2 "github.com/google/cadvisor/info/v2" ) func init() { @@ -105,6 +106,11 @@ func newTestKubelet(t *testing.T) *TestKubelet { kubelet.podManager = podManager kubelet.containerRefManager = kubecontainer.NewRefManager() runtimeHooks := newKubeletRuntimeHooks(kubelet.recorder) + diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, DiskSpacePolicy{}) + if err != nil { + t.Fatalf("can't initialize disk space manager: %v", err) + } + kubelet.diskSpaceManager = diskSpaceManager kubelet.containerRuntime = dockertools.NewFakeDockerManager(fakeDocker, fakeRecorder, kubelet.readinessManager, kubelet.containerRefManager, dockertools.PodInfraContainerImage, 0, 0, "", kubelet.os, kubelet.networkPlugin, kubelet, &fakeHTTP{}, runtimeHooks) kubelet.runtimeCache = kubecontainer.NewFakeRuntimeCache(kubelet.containerRuntime) @@ -365,6 +371,8 @@ func generatePodInfraContainerHash(pod *api.Pod) uint64 { func TestSyncPodsDoesNothing(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -430,6 +438,8 @@ func TestSyncPodsDoesNothing(t *testing.T) { func TestSyncPodsWithTerminationLog(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker container := api.Container{ @@ -492,6 +502,8 @@ func matchString(t *testing.T, pattern, str string) bool { func TestSyncPodsCreatesNetAndContainer(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker // TODO: Move this test to dockertools so that we don't have to do the hacky @@ -556,6 +568,8 @@ func TestSyncPodsCreatesNetAndContainer(t *testing.T) { func TestSyncPodsCreatesNetAndContainerPullsImage(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker // TODO: Move this test to dockertools so that we don't have to do the hacky @@ -616,6 +630,8 @@ func TestSyncPodsCreatesNetAndContainerPullsImage(t *testing.T) { func TestSyncPodsWithPodInfraCreatesContainer(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker pods := []*api.Pod{ @@ -687,6 +703,8 @@ func (f *fakeHTTP) Get(url string) (*http.Response, error) { func TestSyncPodsWithPodInfraCreatesContainerCallsHandler(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker fakeHttp := fakeHTTP{} @@ -769,6 +787,8 @@ func TestSyncPodsWithPodInfraCreatesContainerCallsHandler(t *testing.T) { func TestSyncPodsDeletesWithNoPodInfraContainer(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -884,6 +904,8 @@ func TestSyncPodsDeletesWhenSourcesAreReady(t *testing.T) { ready := false testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker kubelet.sourcesReady = func() bool { return ready } @@ -929,6 +951,8 @@ func TestSyncPodsDeletesWhenSourcesAreReady(t *testing.T) { func TestSyncPodsDeletes(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker fakeDocker.ContainerList = []docker.APIContainers{ @@ -970,6 +994,8 @@ func TestSyncPodsDeletes(t *testing.T) { func TestSyncPodsDeletesDuplicate(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -1051,6 +1077,8 @@ func TestSyncPodsDeletesDuplicate(t *testing.T) { func TestSyncPodsBadHash(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -1122,6 +1150,8 @@ func TestSyncPodsBadHash(t *testing.T) { func TestSyncPodsUnhealthy(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -1640,6 +1670,8 @@ func TestRunInContainer(t *testing.T) { func TestSyncPodEventHandlerFails(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -1727,6 +1759,8 @@ func TestSyncPodEventHandlerFails(t *testing.T) { func TestSyncPodsWithPullPolicy(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker // TODO: Move this test to dockertools so that we don't have to do the hacky @@ -2916,6 +2950,8 @@ func TestHandlePortConflicts(t *testing.T) { testKubelet := newTestKubelet(t) kl := testKubelet.kubelet testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) spec := api.PodSpec{Containers: []api.Container{{Ports: []api.ContainerPort{{HostPort: 80}}}}} pods := []*api.Pod{ @@ -2971,6 +3007,8 @@ func TestHandleNodeSelector(t *testing.T) { {ObjectMeta: api.ObjectMeta{Name: "testnode", Labels: map[string]string{"key": "B"}}}, }} testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) pods := []*api.Pod{ { ObjectMeta: api.ObjectMeta{ @@ -3018,6 +3056,8 @@ func TestHandleMemExceeded(t *testing.T) { testKubelet := newTestKubelet(t) kl := testKubelet.kubelet testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{MemoryCapacity: 100}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) spec := api.PodSpec{Containers: []api.Container{{Resources: api.ResourceRequirements{ Limits: api.ResourceList{ @@ -3073,6 +3113,8 @@ func TestHandleMemExceeded(t *testing.T) { func TestPurgingObsoleteStatusMapEntries(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kl := testKubelet.kubelet pods := []*api.Pod{ @@ -3494,6 +3536,8 @@ func TestCreateMirrorPod(t *testing.T) { func TestDeleteOutdatedMirrorPod(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kl := testKubelet.kubelet manager := testKubelet.fakeMirrorClient pod := &api.Pod{ @@ -3545,6 +3589,8 @@ func TestDeleteOutdatedMirrorPod(t *testing.T) { func TestDeleteOrphanedMirrorPods(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kl := testKubelet.kubelet manager := testKubelet.fakeMirrorClient orphanPods := []*api.Pod{ @@ -3665,6 +3711,8 @@ func TestGetContainerInfoForMirrorPods(t *testing.T) { func TestDoNotCacheStatusForStaticPods(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet pods := []*api.Pod{ @@ -3811,6 +3859,8 @@ func TestPrivilegeContainerDisallowed(t *testing.T) { func TestSyncPodsWithRestartPolicy(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -3970,6 +4020,8 @@ func TestSyncPodsWithRestartPolicy(t *testing.T) { func TestGetPodStatusWithLastTermination(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker @@ -4243,6 +4295,8 @@ func TestGetPodPullImageFailureReason(t *testing.T) { func TestGetRestartCount(t *testing.T) { testKubelet := newTestKubelet(t) testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorApi.MachineInfo{}, nil) + testKubelet.fakeCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) + testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorApiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeDocker := testKubelet.fakeDocker