github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Add external metrics client to HPA rest client

Browse Source
This commit is contained in:
Maciej Pytel 2018-02-21 19:05:26 +01:00
parent 5fcc8dd564
commit 66f4f9080d
10 changed files with 149 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kube-controller-manager/app/BUILD
View File

@ -129,6 +129,7 @@ go_library(
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library",
],
)

2
cmd/kube-controller-manager/app/autoscaling.go
View File

@ -31,6 +31,7 @@ import (
"k8s.io/kubernetes/pkg/controller/podautoscaler/metrics"
resourceclient "k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1"
"k8s.io/metrics/pkg/client/custom_metrics"
"k8s.io/metrics/pkg/client/external_metrics"
)
func startHPAController(ctx ControllerContext) (bool, error) {
@ -51,6 +52,7 @@ func startHPAControllerWithRESTClient(ctx ControllerContext) (bool, error) {
metricsClient := metrics.NewRESTMetricsClient(
resourceclient.NewForConfigOrDie(clientConfig),
custom_metrics.NewForConfigOrDie(clientConfig),
external_metrics.NewForConfigOrDie(clientConfig),
)
return startHPAControllerWithMetricsClient(ctx, metricsClient)
}

1
pkg/controller/podautoscaler/BUILD
View File

@ -86,6 +86,7 @@ go_test(
"//vendor/k8s.io/metrics/pkg/apis/metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/fake:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
],
)

22
pkg/controller/podautoscaler/horizontal_test.go
View File

@ -45,6 +45,7 @@ import (
metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1"
metricsfake "k8s.io/metrics/pkg/client/clientset_generated/clientset/fake"
cmfake "k8s.io/metrics/pkg/client/custom_metrics/fake"
emfake "k8s.io/metrics/pkg/client/external_metrics/fake"
"github.com/stretchr/testify/assert"
@ -145,7 +146,7 @@ func init() {
scaleUpLimitFactor = 8
}
func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *scalefake.FakeScaleClient) {
func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *emfake.FakeExternalMetricsClient, *scalefake.FakeScaleClient) {
namespace := "test-namespace"
hpaName := "test-hpa"
podNamePrefix := "test-pod"
@ -523,7 +524,9 @@ func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfa
return true, metrics, nil
})
return fakeClient, fakeMetricsClient, fakeCMClient, fakeScaleClient
fakeEMClient := &emfake.FakeExternalMetricsClient{}
return fakeClient, fakeMetricsClient, fakeCMClient, fakeEMClient, fakeScaleClient
}
func (tc *testCase) verifyResults(t *testing.T) {
@ -538,7 +541,7 @@ func (tc *testCase) verifyResults(t *testing.T) {
}
func (tc *testCase) setupController(t *testing.T) (*HorizontalController, informers.SharedInformerFactory) {
testClient, testMetricsClient, testCMClient, testScaleClient := tc.prepareTestClient(t)
testClient, testMetricsClient, testCMClient, testEMClient, testScaleClient := tc.prepareTestClient(t)
if tc.testClient != nil {
testClient = tc.testClient
}
@ -554,6 +557,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
metricsClient := metrics.NewRESTMetricsClient(
testMetricsClient.MetricsV1beta1(),
testCMClient,
testEMClient,
)
eventClient := &fake.Clientset{}
@ -1268,7 +1272,7 @@ func TestConditionInvalidSelectorMissing(t *testing.T) {
},
}
_, _, _, testScaleClient := tc.prepareTestClient(t)
_, _, _, _,testScaleClient := tc.prepareTestClient(t)
tc.testScaleClient = testScaleClient
testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -1313,7 +1317,7 @@ func TestConditionInvalidSelectorUnparsable(t *testing.T) {
},
}
_, _, _, testScaleClient := tc.prepareTestClient(t)
_, _, _, _, testScaleClient := tc.prepareTestClient(t)
tc.testScaleClient = testScaleClient
testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -1374,7 +1378,7 @@ func TestConditionFailedGetMetrics(t *testing.T) {
reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
useMetricsAPI: true,
}
_, testMetricsClient, testCMClient, _ := tc.prepareTestClient(t)
_, testMetricsClient, testCMClient, _, _ := tc.prepareTestClient(t)
tc.testMetricsClient = testMetricsClient
tc.testCMClient = testCMClient
@ -1447,7 +1451,7 @@ func TestConditionFailedGetScale(t *testing.T) {
},
}
_, _, _, testScaleClient := tc.prepareTestClient(t)
_, _, _, _, testScaleClient := tc.prepareTestClient(t)
tc.testScaleClient = testScaleClient
testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -1474,7 +1478,7 @@ func TestConditionFailedUpdateScale(t *testing.T) {
}),
}
_, _, _, testScaleClient := tc.prepareTestClient(t)
_, _, _, _, testScaleClient := tc.prepareTestClient(t)
tc.testScaleClient = testScaleClient
testScaleClient.PrependReactor("update", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -1660,7 +1664,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) {
reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
useMetricsAPI: true,
}
testClient, _, _, _ := tc.prepareTestClient(t)
testClient, _, _, _, _ := tc.prepareTestClient(t)
tc.testClient = testClient
var savedHPA *autoscalingv1.HorizontalPodAutoscaler
testClient.PrependReactor("list", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {

3
pkg/controller/podautoscaler/metrics/BUILD
View File

@ -29,6 +29,7 @@ go_library(
"//vendor/k8s.io/metrics/pkg/apis/metrics/v1alpha1:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library",
],
)
@ -56,10 +57,12 @@ go_test(
"//vendor/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library",
"//vendor/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/apis/external_metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/apis/metrics/v1alpha1:go_default_library",
"//vendor/k8s.io/metrics/pkg/apis/metrics/v1beta1:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/fake:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
],
)

4
pkg/controller/podautoscaler/metrics/interfaces.go
View File

@ -42,4 +42,8 @@ type MetricsClient interface {
// GetObjectMetric gets the given metric (and an associated timestamp) for the given
// object in the given namespace
GetObjectMetric(metricName string, namespace string, objectRef *autoscaling.CrossVersionObjectReference) (int64, time.Time, error)
// GetExternalMetric gets all the values of a given external metric
// that match the specified selector.
GetExternalMetric(metricName string, namespace string, selector labels.Selector) ([]int64, time.Time, error)
}

4
pkg/controller/podautoscaler/metrics/legacy_metrics_client.go
View File

@ -177,6 +177,10 @@ func (h *HeapsterMetricsClient) GetObjectMetric(metricName string, namespace str
return 0, time.Time{}, fmt.Errorf("object metrics are not yet supported")
}
func (h *HeapsterMetricsClient) GetExternalMetric(metricName, namespace string, selector labels.Selector) ([]int64, time.Time, error) {
return nil, time.Time{}, fmt.Errorf("external metrics aren't supported")
}
func collapseTimeSamples(metrics heapster.MetricResult, duration time.Duration) (int64, time.Time, bool) {
floatSum := float64(0)
intSum := int64(0)

31
pkg/controller/podautoscaler/metrics/rest_metrics_client.go
View File

@ -30,12 +30,14 @@ import (
customapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1"
resourceclient "k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1"
customclient "k8s.io/metrics/pkg/client/custom_metrics"
externalclient "k8s.io/metrics/pkg/client/external_metrics"
)
func NewRESTMetricsClient(resourceClient resourceclient.PodMetricsesGetter, customClient customclient.CustomMetricsClient) MetricsClient {
func NewRESTMetricsClient(resourceClient resourceclient.PodMetricsesGetter, customClient customclient.CustomMetricsClient, externalClient externalclient.ExternalMetricsClient) MetricsClient {
return &restMetricsClient{
&resourceMetricsClient{resourceClient},
&customMetricsClient{customClient},
&externalMetricsClient{externalClient},
}
}
@ -45,6 +47,7 @@ func NewRESTMetricsClient(resourceClient resourceclient.PodMetricsesGetter, cust
type restMetricsClient struct {
*resourceMetricsClient
*customMetricsClient
*externalMetricsClient
}
// resourceMetricsClient implements the resource-metrics-related parts of MetricsClient,
@ -139,3 +142,29 @@ func (c *customMetricsClient) GetObjectMetric(metricName string, namespace strin
return metricValue.Value.MilliValue(), metricValue.Timestamp.Time, nil
}
// externalMetricsClient implenets the external metrics related parts of MetricsClient,
// using data from the external metrics API.
type externalMetricsClient struct {
client externalclient.ExternalMetricsClient
}
// GetExternalMetric gets all the values of a given external metric
// that match the specified selector.
func (c *externalMetricsClient) GetExternalMetric(metricName, namespace string, selector labels.Selector) ([]int64, time.Time, error) {
metrics, err := c.client.NamespacedMetrics(namespace).Get(metricName, selector)
if err != nil {
return []int64{}, time.Time{}, fmt.Errorf("unable to fetch metrics from external metrics API: %v", err)
}
if len(metrics.Items) == 0 {
return nil, time.Time{}, fmt.Errorf("no metrics returned from external metrics API")
}
res := make([]int64, 0)
for _, m := range metrics.Items {
res = append(res, m.Value.MilliValue())
}
timestamp := metrics.Items[0].Timestamp.Time
return res, timestamp, nil
}

93
pkg/controller/podautoscaler/metrics/rest_metrics_client_test.go
View File

@ -32,9 +32,11 @@ import (
"k8s.io/kubernetes/pkg/api/legacyscheme"
_ "k8s.io/kubernetes/pkg/apis/extensions/install"
cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1"
emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1"
metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1"
metricsfake "k8s.io/metrics/pkg/client/clientset_generated/clientset/fake"
cmfake "k8s.io/metrics/pkg/client/custom_metrics/fake"
emfake "k8s.io/metrics/pkg/client/external_metrics/fake"
"github.com/stretchr/testify/assert"
)
@ -49,13 +51,15 @@ type restClientTestCase struct {
reportedPodMetrics [][]int64
singleObject *autoscalingapi.CrossVersionObjectReference
namespace string
selector labels.Selector
resourceName v1.ResourceName
metricName string
namespace string
selector labels.Selector
resourceName v1.ResourceName
metricName string
metricSelector *metav1.LabelSelector
metricLabelSelector labels.Selector
}
func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clientset, *cmfake.FakeCustomMetricsClient) {
func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *emfake.FakeExternalMetricsClient) {
namespace := "test-namespace"
tc.namespace = namespace
podNamePrefix := "test-pod"
@ -64,9 +68,12 @@ func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clie
// it's a resource test if we have a resource name
isResource := len(tc.resourceName) > 0
// it's an external test if we have a metric selector
isExternal := tc.metricSelector != nil
fakeMetricsClient := &metricsfake.Clientset{}
fakeCMClient := &cmfake.FakeCustomMetricsClient{}
fakeEMClient := &emfake.FakeExternalMetricsClient{}
if isResource {
fakeMetricsClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -99,6 +106,24 @@ func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clie
}
return true, metrics, nil
})
} else if isExternal {
fakeEMClient.AddReactor("get", "externalmetrics", func(action core.Action) (handled bool, ret runtime.Object, err error) {
getForAction := action.(emfake.GetForAction)
assert.Equal(t, tc.metricName, getForAction.GetMetricName(), "the metric requested should have matched the one specified.")
assert.Equal(t, tc.metricLabelSelector, getForAction.GetMetricSelector(), "the metric selector should have matched the one specified")
metrics := emapi.ExternalMetricValueList{}
for _, metricPoint := range tc.reportedMetricPoints {
timestamp := fixedTimestamp.Add(time.Duration(metricPoint.timestamp) * time.Minute)
metric := emapi.ExternalMetricValue{
Value: *resource.NewMilliQuantity(int64(metricPoint.level), resource.DecimalSI),
Timestamp: metav1.Time{Time: timestamp},
MetricName: tc.metricName,
}
metrics.Items = append(metrics.Items, metric)
}
return true, &metrics, nil
})
} else {
fakeCMClient.AddReactor("get", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
getForAction := action.(cmfake.GetForAction)
@ -162,13 +187,13 @@ func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clie
})
}
return fakeMetricsClient, fakeCMClient
return fakeMetricsClient, fakeCMClient, fakeEMClient
}
func (tc *restClientTestCase) verifyResults(t *testing.T, metrics PodMetricsInfo, timestamp time.Time, err error) {
if tc.desiredError != nil {
assert.Error(t, err, "there should be an error retrieving the metrics")
assert.Contains(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.desiredError), "the error message should be eas expected")
assert.Contains(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.desiredError), "the error message should be as expected")
return
}
assert.NoError(t, err, "there should be no error retrieving the metrics")
@ -181,12 +206,25 @@ func (tc *restClientTestCase) verifyResults(t *testing.T, metrics PodMetricsInfo
}
func (tc *restClientTestCase) runTest(t *testing.T) {
testMetricsClient, testCMClient := tc.prepareTestClient(t)
metricsClient := NewRESTMetricsClient(testMetricsClient.MetricsV1beta1(), testCMClient)
var err error
testMetricsClient, testCMClient, testEMClient := tc.prepareTestClient(t)
metricsClient := NewRESTMetricsClient(testMetricsClient.MetricsV1beta1(), testCMClient, testEMClient)
isResource := len(tc.resourceName) > 0
isExternal := tc.metricSelector != nil
if isResource {
info, timestamp, err := metricsClient.GetResourceMetric(v1.ResourceName(tc.resourceName), tc.namespace, tc.selector)
tc.verifyResults(t, info, timestamp, err)
} else if isExternal {
tc.metricLabelSelector, err = metav1.LabelSelectorAsSelector(tc.metricSelector)
if err != nil {
t.Errorf("invalid metric selector: %+v", tc.metricSelector)
}
val, timestamp, err := metricsClient.GetExternalMetric(tc.metricName, tc.namespace, tc.metricLabelSelector)
info := make(PodMetricsInfo, len(val))
for i, metricVal := range val {
info[fmt.Sprintf("%v-val-%v", tc.metricName, i)] = metricVal
}
tc.verifyResults(t, info, timestamp, err)
} else if tc.singleObject == nil {
info, timestamp, err := metricsClient.GetRawMetric(tc.metricName, tc.namespace, tc.selector)
tc.verifyResults(t, info, timestamp, err)
@ -209,6 +247,19 @@ func TestRESTClientCPU(t *testing.T) {
tc.runTest(t)
}
func TestRESTClientExternal(t *testing.T) {
tc := restClientTestCase{
desiredMetricValues: PodMetricsInfo{
"external-val-0": 10000, "external-val-1": 20000, "external-val-2": 10000,
},
metricSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}},
metricName: "external",
targetTimestamp: 1,
reportedMetricPoints: []metricPoint{{10000, 1}, {20000, 1}, {10000, 1}},
}
tc.runTest(t)
}
func TestRESTClientQPS(t *testing.T) {
tc := restClientTestCase{
desiredMetricValues: PodMetricsInfo{
@ -248,6 +299,19 @@ func TestRESTClientQpsSumEqualZero(t *testing.T) {
tc.runTest(t)
}
func TestRESTClientExternalSumEqualZero(t *testing.T) {
tc := restClientTestCase{
desiredMetricValues: PodMetricsInfo{
"external-val-0": 0, "external-val-1": 0, "external-val-2": 0,
},
metricSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}},
metricName: "external",
targetTimestamp: 0,
reportedMetricPoints: []metricPoint{{0, 0}, {0, 0}, {0, 0}},
}
tc.runTest(t)
}
func TestRESTClientQpsEmptyMetrics(t *testing.T) {
tc := restClientTestCase{
metricName: "qps",
@ -258,6 +322,17 @@ func TestRESTClientQpsEmptyMetrics(t *testing.T) {
tc.runTest(t)
}
func TestRESTClientExternalEmptyMetrics(t *testing.T) {
tc := restClientTestCase{
metricName: "external",
metricSelector: &metav1.LabelSelector{},
desiredError: fmt.Errorf("no metrics returned from external metrics API"),
reportedMetricPoints: []metricPoint{},
}
tc.runTest(t)
}
func TestRESTClientCPUEmptyMetrics(t *testing.T) {
tc := restClientTestCase{
resourceName: v1.ResourceCPU,

11
pkg/controller/podautoscaler/replica_calculator_test.go
View File

@ -36,6 +36,7 @@ import (
metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1"
metricsfake "k8s.io/metrics/pkg/client/clientset_generated/clientset/fake"
cmfake "k8s.io/metrics/pkg/client/custom_metrics/fake"
emfake "k8s.io/metrics/pkg/client/external_metrics/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -81,7 +82,7 @@ const (
numContainersPerPod = 2
)
func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfake.Clientset, *cmfake.FakeCustomMetricsClient) {
func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *emfake.FakeExternalMetricsClient) {
fakeClient := &fake.Clientset{}
fakeClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
@ -236,12 +237,14 @@ func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) (*fake.Clientset,
return true, metrics, nil
})
return fakeClient, fakeMetricsClient, fakeCMClient
fakeEMClient := &emfake.FakeExternalMetricsClient{}
return fakeClient, fakeMetricsClient, fakeCMClient, fakeEMClient
}
func (tc *replicaCalcTestCase) runTest(t *testing.T) {
testClient, testMetricsClient, testCMClient := tc.prepareTestClient(t)
metricsClient := metrics.NewRESTMetricsClient(testMetricsClient.MetricsV1beta1(), testCMClient)
testClient, testMetricsClient, testCMClient, testEMClient := tc.prepareTestClient(t)
metricsClient := metrics.NewRESTMetricsClient(testMetricsClient.MetricsV1beta1(), testCMClient, testEMClient)
replicaCalc := &ReplicaCalculator{
metricsClient: metricsClient,