diff --git a/pkg/kubectl/cmd/top_node.go b/pkg/kubectl/cmd/top_node.go index 22604af092c..b685b5649e9 100644 --- a/pkg/kubectl/cmd/top_node.go +++ b/pkg/kubectl/cmd/top_node.go @@ -107,11 +107,7 @@ func (o *TopNodeOptions) Validate() error { } func (o TopNodeOptions) RunTopNode() error { - metrics, err := o.Client.GetNodeMetrics(o.ResourceName, o.Selector) - if err != nil { - return err - } - + var err error selector := labels.Everything() if len(o.Selector) > 0 { selector, err = labels.Parse(o.Selector) @@ -119,6 +115,11 @@ func (o TopNodeOptions) RunTopNode() error { return err } } + metrics, err := o.Client.GetNodeMetrics(o.ResourceName, selector) + if err != nil { + return err + } + var nodes []api.Node if len(o.ResourceName) > 0 { node, err := o.Client.Nodes().Get(o.ResourceName) diff --git a/pkg/kubectl/cmd/top_pod.go b/pkg/kubectl/cmd/top_pod.go index eff06c39cc4..c2549791582 100644 --- a/pkg/kubectl/cmd/top_pod.go +++ b/pkg/kubectl/cmd/top_pod.go @@ -18,31 +18,40 @@ package cmd import ( "errors" + "fmt" "io" + "time" + "k8s.io/kubernetes/pkg/api" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/metricsutil" + "k8s.io/kubernetes/pkg/labels" + "github.com/golang/glog" "github.com/renstrom/dedent" "github.com/spf13/cobra" ) -// TopPodOptions contains all the options for running the top-pod cli command. type TopPodOptions struct { ResourceName string + Namespace string + Selector string AllNamespaces bool PrintContainers bool - Selector string - Namespace string Client *metricsutil.HeapsterMetricsClient Printer *metricsutil.TopCmdPrinter } +const metricsCreationDelay = 2 * time.Minute + var ( topPodLong = dedent.Dedent(` Display Resource (CPU/Memory/Storage) usage of pods. - The 'top pod' command allows you to see the resource consumption of pods.`) + The 'top pod' command allows you to see the resource consumption of pods. + + Due to the metrics pipeline delay, they may be unavailable for a few minutes + since pod creation.`) topPodExample = dedent.Dedent(` # Show metrics for all pods in the default namespace @@ -82,7 +91,6 @@ func NewCmdTopPod(f *cmdutil.Factory, out io.Writer) *cobra.Command { return cmd } -// Complete completes all the required options for top. func (o *TopPodOptions) Complete(f *cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error { var err error if len(args) == 1 { @@ -111,11 +119,68 @@ func (o *TopPodOptions) Validate() error { return nil } -// RunTop implements all the necessary functionality for top. func (o TopPodOptions) RunTopPod() error { - metrics, err := o.Client.GetPodMetrics(o.Namespace, o.ResourceName, o.AllNamespaces, o.Selector) + var err error + selector := labels.Everything() + if len(o.Selector) > 0 { + selector, err = labels.Parse(o.Selector) + if err != nil { + return err + } + } + metrics, err := o.Client.GetPodMetrics(o.Namespace, o.ResourceName, o.AllNamespaces, selector) + // TODO: Refactor this once Heapster becomes the API server. + // First we check why no metrics have been received. + if len(metrics) == 0 { + // If the API server query is successful but all the pods are newly created, + // the metrics are probably not ready yet, so we return the error here in the first place. + e := verifyEmptyMetrics(o, selector) + if e != nil { + return e + } + } if err != nil { return err } return o.Printer.PrintPodMetrics(metrics, o.PrintContainers, o.AllNamespaces) } + +func verifyEmptyMetrics(o TopPodOptions, selector labels.Selector) error { + if len(o.ResourceName) > 0 { + pod, err := o.Client.Pods(o.Namespace).Get(o.ResourceName) + if err != nil { + return err + } + if err := checkPodAge(pod); err != nil { + return err + } + } else { + pods, err := o.Client.Pods(o.Namespace).List(api.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return err + } + if len(pods.Items) == 0 { + return nil + } + for _, pod := range pods.Items { + if err := checkPodAge(&pod); err != nil { + return err + } + } + } + return errors.New("metrics not available yet") +} + +func checkPodAge(pod *api.Pod) error { + age := time.Since(pod.CreationTimestamp.Time) + if age > metricsCreationDelay { + message := fmt.Sprintf("Metrics not available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String()) + glog.Warningf(message) + return errors.New(message) + } else { + glog.V(2).Infof("Metrics not yet available for pod %s/%s, age: %s", pod.Namespace, pod.Name, age.String()) + return nil + } +} diff --git a/pkg/kubectl/metricsutil/metrics_client.go b/pkg/kubectl/metricsutil/metrics_client.go index 23266c73717..1dce9e00b62 100644 --- a/pkg/kubectl/metricsutil/metrics_client.go +++ b/pkg/kubectl/metricsutil/metrics_client.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/api/validation" client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/labels" ) const ( @@ -96,8 +97,8 @@ func nodeMetricsUrl(name string) (string, error) { return fmt.Sprintf("%s/nodes/%s", metricsRoot, name), nil } -func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector string) ([]metrics_api.NodeMetrics, error) { - params := map[string]string{"labelSelector": selector} +func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector labels.Selector) ([]metrics_api.NodeMetrics, error) { + params := map[string]string{"labelSelector": selector.String()} path, err := nodeMetricsUrl(nodeName) if err != nil { return []metrics_api.NodeMetrics{}, err @@ -125,7 +126,7 @@ func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector strin return metrics, nil } -func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string, allNamespaces bool, selector string) ([]metrics_api.PodMetrics, error) { +func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string, allNamespaces bool, selector labels.Selector) ([]metrics_api.PodMetrics, error) { if allNamespaces { namespace = api.NamespaceAll } @@ -133,7 +134,8 @@ func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string if err != nil { return []metrics_api.PodMetrics{}, err } - params := map[string]string{"labelSelector": selector} + + params := map[string]string{"labelSelector": selector.String()} allMetrics := make([]metrics_api.PodMetrics, 0) resultRaw, err := GetHeapsterMetrics(cli, path, params)