Merge pull request #64170 from mtaufen/cap-node-num-images

Automatic merge from submit-queue (batch tested with PRs 61803, 64305, 64170, 64361, 64339). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

add a flag to control the cap on images reported in node status

While I normally try to avoid adding flags, this is a short term
scalability fix for v1.11, and there are other long-term solutions in
the works, so we shouldn't commit to this in the v1beta1 Kubelet config.
Flags are our escape hatch here.

```release-note
NONE
```

cmd/kubelet/app/options/options.go

@@ -169,6 +169,9 @@ type KubeletFlags struct {
 	// bootstrapCheckpointPath is the path to the directory containing pod checkpoints to
 	// run on restore
 	BootstrapCheckpointPath string
+	// NodeStatusMaxImages caps the number of images reported in Node.Status.Images.
+	// This is an experimental, short-term flag to help with node scalability.
+	NodeStatusMaxImages int32
 
 	// DEPRECATED FLAGS
 	// minimumGCAge is the minimum age for a finished container before it is
@@ -244,6 +247,8 @@ func NewKubeletFlags() *KubeletFlags {
 		CAdvisorPort: 0,
 		// TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated
 		AllowPrivileged: true,
+		// prior to the introduction of this flag, there was a hardcoded cap of 50 images
+		NodeStatusMaxImages: 50,
 	}
 }
 
@@ -255,6 +260,9 @@ func ValidateKubeletFlags(f *KubeletFlags) error {
 	if f.CAdvisorPort != 0 && utilvalidation.IsValidPortNum(int(f.CAdvisorPort)) != nil {
 		return fmt.Errorf("invalid configuration: CAdvisorPort (--cadvisor-port) %v must be between 0 and 65535, inclusive", f.CAdvisorPort)
 	}
+	if f.NodeStatusMaxImages < -1 {
+		return fmt.Errorf("invalid configuration: NodeStatusMaxImages (--node-status-max-images) must be -1 or greater")
+	}
 	return nil
 }
 
@@ -392,6 +400,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
 	fs.BoolVar(&f.ExitOnLockContention, "exit-on-lock-contention", f.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
 	fs.StringVar(&f.SeccompProfileRoot, "seccomp-profile-root", f.SeccompProfileRoot, "<Warning: Alpha feature> Directory path for seccomp profiles.")
 	fs.StringVar(&f.BootstrapCheckpointPath, "bootstrap-checkpoint-path", f.BootstrapCheckpointPath, "<Warning: Alpha feature> Path to to the directory where the checkpoints are stored")
+	fs.Int32Var(&f.NodeStatusMaxImages, "node-status-max-images", f.NodeStatusMaxImages, "<Warning: Alpha feature> The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied. Default: 50")
 
 	// DEPRECATED FLAGS
 	fs.StringVar(&f.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", f.BootstrapKubeconfig, "")

cmd/kubelet/app/server.go

@@ -958,6 +958,7 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *kubeletconfiginternal.
 		kubeFlags.NodeLabels,
 		kubeFlags.SeccompProfileRoot,
 		kubeFlags.BootstrapCheckpointPath,
+		kubeFlags.NodeStatusMaxImages,
 		stopCh)
 	if err != nil {
 		return fmt.Errorf("failed to create kubelet: %v", err)
@@ -1043,6 +1044,7 @@ func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	nodeLabels map[string]string,
 	seccompProfileRoot string,
 	bootstrapCheckpointPath string,
+	nodeStatusMaxImages int32,
 	stopCh <-chan struct{}) (k kubelet.Bootstrap, err error) {
 	// TODO: block until all sources have delivered at least one update to the channel, or break the sync loop
 	// up into "per source" synchronizations
@@ -1077,6 +1079,7 @@ func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		nodeLabels,
 		seccompProfileRoot,
 		bootstrapCheckpointPath,
+		nodeStatusMaxImages,
 		stopCh)
 	if err != nil {
 		return nil, err

pkg/kubelet/kubelet.go

@@ -219,7 +219,9 @@ type Builder func(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	keepTerminatedPodVolumes bool,
 	nodeLabels map[string]string,
 	seccompProfileRoot string,
-	bootstrapCheckpointPath string) (Bootstrap, error)
+	bootstrapCheckpointPath string,
+	nodeStatusMaxImages int32,
+	stopCh <-chan struct{}) (Bootstrap, error)
 
 // Dependencies is a bin for things we might consider "injected dependencies" -- objects constructed
 // at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping
@@ -345,6 +347,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	nodeLabels map[string]string,
 	seccompProfileRoot string,
 	bootstrapCheckpointPath string,
+	nodeStatusMaxImages int32,
 	stopCh <-chan struct{}) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@@ -535,6 +538,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		iptablesDropBit:                         int(kubeCfg.IPTablesDropBit),
 		experimentalHostUserNamespaceDefaulting: utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalHostUserNamespaceDefaultingGate),
 		keepTerminatedPodVolumes:                keepTerminatedPodVolumes,
+		nodeStatusMaxImages:                     nodeStatusMaxImages,
 	}
 
 	if klet.cloud != nil {
@@ -1157,6 +1161,9 @@ type Kubelet struct {
 	// such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the
 	// directory returned by kubelet.getPluginsDir()
 	pluginWatcher pluginwatcher.Watcher
+
+	// This flag sets a maximum number of images to report in the node status.
+	nodeStatusMaxImages int32
 }
 
 func allGlobalUnicastIPs() ([]net.IP, error) {

pkg/kubelet/kubelet_node_status.go

@@ -49,9 +49,6 @@ import (
 )
 
 const (
-	// maxImagesInNodeStatus is the number of max images we store in image status.
-	maxImagesInNodeStatus = 50
-
 	// maxNamesPerImageInNodeStatus is max number of names per image stored in
 	// the node status.
 	maxNamesPerImageInNodeStatus = 5
@@ -721,8 +718,9 @@ func (kl *Kubelet) setNodeStatusImages(node *v1.Node) {
 		return
 	}
 	// sort the images from max to min, and only set top N images into the node status.
-	if maxImagesInNodeStatus < len(containerImages) {
+	if int(kl.nodeStatusMaxImages) > -1 &&
-		containerImages = containerImages[0:maxImagesInNodeStatus]
+		int(kl.nodeStatusMaxImages) < len(containerImages) {
+		containerImages = containerImages[0:kl.nodeStatusMaxImages]
 	}
 
 	for _, image := range containerImages {

pkg/kubelet/kubelet_node_status_test.go

@@ -60,8 +60,8 @@ const (
 	maxImageTagsForTest = 20
 )
 
-// generateTestingImageList generate randomly generated image list and corresponding expectedImageList.
+// generateTestingImageLists generate randomly generated image list and corresponding expectedImageList.
-func generateTestingImageList(count int) ([]kubecontainer.Image, []v1.ContainerImage) {
+func generateTestingImageLists(count int, maxImages int) ([]kubecontainer.Image, []v1.ContainerImage) {
 	// imageList is randomly generated image list
 	var imageList []kubecontainer.Image
 	for ; count > 0; count-- {
@@ -73,7 +73,12 @@ func generateTestingImageList(count int) ([]kubecontainer.Image, []v1.ContainerI
 		imageList = append(imageList, imageItem)
 	}
 
-	// expectedImageList is generated by imageList according to size and maxImagesInNodeStatus
+	expectedImageList := makeExpectedImageList(imageList, maxImages)
+	return imageList, expectedImageList
+}
+
+func makeExpectedImageList(imageList []kubecontainer.Image, maxImages int) []v1.ContainerImage {
+	// expectedImageList is generated by imageList according to size and maxImages
 	// 1. sort the imageList by size
 	sort.Sort(sliceutils.ByImageSize(imageList))
 	// 2. convert sorted imageList to v1.ContainerImage list
@@ -86,8 +91,11 @@ func generateTestingImageList(count int) ([]kubecontainer.Image, []v1.ContainerI
 
 		expectedImageList = append(expectedImageList, apiImage)
 	}
-	// 3. only returns the top maxImagesInNodeStatus images in expectedImageList
+	// 3. only returns the top maxImages images in expectedImageList
-	return imageList, expectedImageList[0:maxImagesInNodeStatus]
+	if maxImages == -1 { // -1 means no limit
+		return expectedImageList
+	}
+	return expectedImageList[0:maxImages]
 }
 
 func generateImageTags() []string {
@@ -299,12 +307,34 @@ func sortNodeAddresses(addrs sortableNodeAddress) {
 }
 
 func TestUpdateNewNodeStatus(t *testing.T) {
-	// generate one more than maxImagesInNodeStatus in inputImageList
+	cases := []struct {
-	inputImageList, expectedImageList := generateTestingImageList(maxImagesInNodeStatus + 1)
+		desc                string
+		nodeStatusMaxImages int32
+	}{
+		{
+			desc:                "5 image limit",
+			nodeStatusMaxImages: 5,
+		},
+		{
+			desc:                "no image limit",
+			nodeStatusMaxImages: -1,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.desc, func(t *testing.T) {
+			// generate one more in inputImageList than we configure the Kubelet to report,
+			// or 5 images if unlimited
+			numTestImages := int(tc.nodeStatusMaxImages) + 1
+			if tc.nodeStatusMaxImages == -1 {
+				numTestImages = 5
+			}
+			inputImageList, expectedImageList := generateTestingImageLists(numTestImages, int(tc.nodeStatusMaxImages))
 			testKubelet := newTestKubeletWithImageList(
 				t, inputImageList, false /* controllerAttachDetachEnabled */)
 			defer testKubelet.Cleanup()
 			kubelet := testKubelet.kubelet
+			kubelet.nodeStatusMaxImages = tc.nodeStatusMaxImages
 			kubelet.kubeClient = nil // ensure only the heartbeat client is used
 			kubelet.containerManager = &localCM{
 				ContainerManager: cm.NewStubContainerManager(),
@@ -449,14 +479,17 @@ func TestUpdateNewNodeStatus(t *testing.T) {
 			// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
 			assert.Equal(t, v1.NodeReady, updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type,
 				"NotReady should be last")
-	assert.Len(t, updatedNode.Status.Images, maxImagesInNodeStatus)
+			assert.Len(t, updatedNode.Status.Images, len(expectedImageList))
 			assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
+		})
+	}
 }
 
 func TestUpdateExistingNodeStatus(t *testing.T) {
 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
 	defer testKubelet.Cleanup()
 	kubelet := testKubelet.kubelet
+	kubelet.nodeStatusMaxImages = 5 // don't truncate the image list that gets constructed by hand for this test
 	kubelet.kubeClient = nil        // ensure only the heartbeat client is used
 	kubelet.containerManager = &localCM{
 		ContainerManager: cm.NewStubContainerManager(),
@@ -742,6 +775,7 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
 	defer testKubelet.Cleanup()
 	kubelet := testKubelet.kubelet
+	kubelet.nodeStatusMaxImages = 5 // don't truncate the image list that gets constructed by hand for this test
 	kubelet.kubeClient = nil        // ensure only the heartbeat client is used
 	kubelet.containerManager = &localCM{
 		ContainerManager: cm.NewStubContainerManager(),
@@ -1213,12 +1247,15 @@ func TestTryRegisterWithApiServer(t *testing.T) {
 }
 
 func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) {
-	// generate one more than maxImagesInNodeStatus in inputImageList
+	const nodeStatusMaxImages = 5
-	inputImageList, _ := generateTestingImageList(maxImagesInNodeStatus + 1)
+
+	// generate one more in inputImageList than we configure the Kubelet to report
+	inputImageList, _ := generateTestingImageLists(nodeStatusMaxImages+1, nodeStatusMaxImages)
 	testKubelet := newTestKubeletWithImageList(
 		t, inputImageList, false /* controllerAttachDetachEnabled */)
 	defer testKubelet.Cleanup()
 	kubelet := testKubelet.kubelet
+	kubelet.nodeStatusMaxImages = nodeStatusMaxImages
 	kubelet.kubeClient = nil // ensure only the heartbeat client is used
 	kubelet.containerManager = &localCM{
 		ContainerManager: cm.NewStubContainerManager(),