Merge pull request #31510 from mksalawa/kubectltop

Automatic merge from submit-queue

Print a message if metrics are not yet available.

**What this PR does / why we need it**:
It takes about 80s to gather first metrics for the newly created pod. We would like to indicate this to the user, so that they don't mistake it for the command failure.

In case no metrics are found, we check whether there should be any, and if yes, print the appropriate message to the user.

**Which issue this PR fixes**
#30826 

```release-note
NONE
```

pkg/kubectl/cmd/top_node.go

@@ -107,11 +107,7 @@ func (o *TopNodeOptions) Validate() error {
 }
 
 func (o TopNodeOptions) RunTopNode() error {
-	metrics, err := o.Client.GetNodeMetrics(o.ResourceName, o.Selector)
-	if err != nil {
-		return err
-	}
-
+	var err error
 	selector := labels.Everything()
 	if len(o.Selector) > 0 {
 		selector, err = labels.Parse(o.Selector)
@@ -119,6 +115,11 @@ func (o TopNodeOptions) RunTopNode() error {
 			return err
 		}
 	}
+	metrics, err := o.Client.GetNodeMetrics(o.ResourceName, selector)
+	if err != nil {
+		return err
+	}
+
 	var nodes []api.Node
 	if len(o.ResourceName) > 0 {
 		node, err := o.Client.Nodes().Get(o.ResourceName)

pkg/kubectl/cmd/top_pod.go

@@ -18,31 +18,40 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
 	"io"
+	"time"
 
+	"k8s.io/kubernetes/pkg/api"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 	"k8s.io/kubernetes/pkg/kubectl/metricsutil"
+	"k8s.io/kubernetes/pkg/labels"
 
+	"github.com/golang/glog"
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
 )
 
-// TopPodOptions contains all the options for running the top-pod cli command.
 type TopPodOptions struct {
 	ResourceName    string
+	Namespace       string
+	Selector        string
 	AllNamespaces   bool
 	PrintContainers bool
-	Selector        string
-	Namespace       string
 	Client          *metricsutil.HeapsterMetricsClient
 	Printer         *metricsutil.TopCmdPrinter
 }
 
+const metricsCreationDelay = 2 * time.Minute
+
 var (
 	topPodLong = dedent.Dedent(`
 		Display Resource (CPU/Memory/Storage) usage of pods.
 
-		The 'top pod' command allows you to see the resource consumption of pods.`)
+		The 'top pod' command allows you to see the resource consumption of pods.
+
+		Due to the metrics pipeline delay, they may be unavailable for a few minutes
+		since pod creation.`)
 
 	topPodExample = dedent.Dedent(`
 		  # Show metrics for all pods in the default namespace
@@ -82,7 +91,6 @@ func NewCmdTopPod(f *cmdutil.Factory, out io.Writer) *cobra.Command {
 	return cmd
 }
 
-// Complete completes all the required options for top.
 func (o *TopPodOptions) Complete(f *cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error {
 	var err error
 	if len(args) == 1 {
@@ -111,11 +119,68 @@ func (o *TopPodOptions) Validate() error {
 	return nil
 }
 
-// RunTop implements all the necessary functionality for top.
 func (o TopPodOptions) RunTopPod() error {
-	metrics, err := o.Client.GetPodMetrics(o.Namespace, o.ResourceName, o.AllNamespaces, o.Selector)
+	var err error
+	selector := labels.Everything()
+	if len(o.Selector) > 0 {
+		selector, err = labels.Parse(o.Selector)
+		if err != nil {
+			return err
+		}
+	}
+	metrics, err := o.Client.GetPodMetrics(o.Namespace, o.ResourceName, o.AllNamespaces, selector)
+	// TODO: Refactor this once Heapster becomes the API server.
+	// First we check why no metrics have been received.
+	if len(metrics) == 0 {
+		// If the API server query is successful but all the pods are newly created,
+		// the metrics are probably not ready yet, so we return the error here in the first place.
+		e := verifyEmptyMetrics(o, selector)
+		if e != nil {
+			return e
+		}
+	}
 	if err != nil {
 		return err
 	}
 	return o.Printer.PrintPodMetrics(metrics, o.PrintContainers, o.AllNamespaces)
 }
+
+func verifyEmptyMetrics(o TopPodOptions, selector labels.Selector) error {
+	if len(o.ResourceName) > 0 {
+		pod, err := o.Client.Pods(o.Namespace).Get(o.ResourceName)
+		if err != nil {
+			return err
+		}
+		if err := checkPodAge(pod); err != nil {
+			return err
+		}
+	} else {
+		pods, err := o.Client.Pods(o.Namespace).List(api.ListOptions{
+			LabelSelector: selector,
+		})
+		if err != nil {
+			return err
+		}
+		if len(pods.Items) == 0 {
+			return nil
+		}
+		for _, pod := range pods.Items {
+			if err := checkPodAge(&pod); err != nil {
+				return err
+			}
+		}
+	}
+	return errors.New("metrics not available yet")
+}
+
+func checkPodAge(pod *api.Pod) error {
+	age := time.Since(pod.CreationTimestamp.Time)
+	if age > metricsCreationDelay {
+		message := fmt.Sprintf("Metrics not available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String())
+		glog.Warningf(message)
+		return errors.New(message)
+	} else {
+		glog.V(2).Infof("Metrics not yet available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String())
+		return nil
+	}
+}

pkg/kubectl/metricsutil/metrics_client.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/api/validation"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/labels"
 )
 
 const (
@@ -96,8 +97,8 @@ func nodeMetricsUrl(name string) (string, error) {
 	return fmt.Sprintf("%s/nodes/%s", metricsRoot, name), nil
 }
 
-func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector string) ([]metrics_api.NodeMetrics, error) {
-	params := map[string]string{"labelSelector": selector}
+func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector labels.Selector) ([]metrics_api.NodeMetrics, error) {
+	params := map[string]string{"labelSelector": selector.String()}
 	path, err := nodeMetricsUrl(nodeName)
 	if err != nil {
 		return []metrics_api.NodeMetrics{}, err
@@ -125,7 +126,7 @@ func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector strin
 	return metrics, nil
 }
 
-func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string, allNamespaces bool, selector string) ([]metrics_api.PodMetrics, error) {
+func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string, allNamespaces bool, selector labels.Selector) ([]metrics_api.PodMetrics, error) {
 	if allNamespaces {
 		namespace = api.NamespaceAll
 	}
@@ -133,7 +134,8 @@ func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string
 	if err != nil {
 		return []metrics_api.PodMetrics{}, err
 	}
-	params := map[string]string{"labelSelector": selector}
+
+	params := map[string]string{"labelSelector": selector.String()}
 	allMetrics := make([]metrics_api.PodMetrics, 0)
 
 	resultRaw, err := GetHeapsterMetrics(cli, path, params)