diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index 9d0b9079c20..09cdfd12d41 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -51,7 +51,6 @@ import ( deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" "k8s.io/kubernetes/pkg/fieldpath" "k8s.io/kubernetes/pkg/fields" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/types" @@ -538,7 +537,7 @@ func describePod(pod *api.Pod, events *api.EventList) (string, error) { } } describeVolumes(pod.Spec.Volumes, w, "") - w.Write(LEVEL_0, "QoS Class:\t%s\n", qos.InternalGetPodQOS(pod)) + w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass) printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector) printTolerationsInAnnotationMultiline(w, "Tolerations", pod.Annotations) if events != nil { diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index c649db46191..a1fdb076a32 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -72,6 +72,7 @@ go_library( "//pkg/kubelet/pod:go_default_library", "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/prober/results:go_default_library", + "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/remote:go_default_library", "//pkg/kubelet/rkt:go_default_library", "//pkg/kubelet/server:go_default_library", diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 571b2e6d675..ad261fdde59 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -43,6 +43,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/envvars" "k8s.io/kubernetes/pkg/kubelet/images" + "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/status" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -1120,6 +1121,8 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po func (kl *Kubelet) convertStatusToAPIStatus(pod *v1.Pod, podStatus *kubecontainer.PodStatus) *v1.PodStatus { var apiPodStatus v1.PodStatus apiPodStatus.PodIP = podStatus.IP + // set status for Pods created on versions of kube older than 1.6 + apiPodStatus.QOSClass = qos.GetPodQOS(pod) apiPodStatus.ContainerStatuses = kl.convertToAPIContainerStatuses( pod, podStatus, diff --git a/pkg/kubelet/qos/BUILD b/pkg/kubelet/qos/BUILD index 55e02159748..4a682231944 100644 --- a/pkg/kubelet/qos/BUILD +++ b/pkg/kubelet/qos/BUILD @@ -14,7 +14,6 @@ go_library( "doc.go", "policy.go", "qos.go", - "types.go", ], tags = ["automanaged"], deps = [ diff --git a/pkg/registry/core/pod/BUILD b/pkg/registry/core/pod/BUILD index 9744afac9ae..8fdd739d5e5 100644 --- a/pkg/registry/core/pod/BUILD +++ b/pkg/registry/core/pod/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/fields:go_default_library", "//pkg/genericapiserver/api/request:go_default_library", "//pkg/kubelet/client:go_default_library", + "//pkg/kubelet/qos:go_default_library", "//pkg/labels:go_default_library", "//pkg/registry/generic:go_default_library", "//pkg/runtime:go_default_library", @@ -41,6 +42,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/api/errors:go_default_library", + "//pkg/api/resource:go_default_library", "//pkg/api/testing:go_default_library", "//pkg/apimachinery/registered:go_default_library", "//pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/core/pod/strategy.go b/pkg/registry/core/pod/strategy.go index 92320b011f5..16c61394989 100644 --- a/pkg/registry/core/pod/strategy.go +++ b/pkg/registry/core/pod/strategy.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/fields" genericapirequest "k8s.io/kubernetes/pkg/genericapiserver/api/request" "k8s.io/kubernetes/pkg/kubelet/client" + "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/registry/generic" "k8s.io/kubernetes/pkg/runtime" @@ -60,7 +61,8 @@ func (podStrategy) NamespaceScoped() bool { func (podStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) { pod := obj.(*api.Pod) pod.Status = api.PodStatus{ - Phase: api.PodPending, + Phase: api.PodPending, + QOSClass: qos.InternalGetPodQOS(pod), } } diff --git a/pkg/registry/core/pod/strategy_test.go b/pkg/registry/core/pod/strategy_test.go index b1550ac42ab..37d0b5b9583 100644 --- a/pkg/registry/core/pod/strategy_test.go +++ b/pkg/registry/core/pod/strategy_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/api/resource" apitesting "k8s.io/kubernetes/pkg/api/testing" "k8s.io/kubernetes/pkg/apimachinery/registered" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" @@ -92,6 +93,80 @@ func TestMatchPod(t *testing.T) { } } +func getResourceList(cpu, memory string) api.ResourceList { + res := api.ResourceList{} + if cpu != "" { + res[api.ResourceCPU] = resource.MustParse(cpu) + } + if memory != "" { + res[api.ResourceMemory] = resource.MustParse(memory) + } + return res +} + +func addResource(rName, value string, rl api.ResourceList) api.ResourceList { + rl[api.ResourceName(rName)] = resource.MustParse(value) + return rl +} + +func getResourceRequirements(requests, limits api.ResourceList) api.ResourceRequirements { + res := api.ResourceRequirements{} + res.Requests = requests + res.Limits = limits + return res +} + +func newContainer(name string, requests api.ResourceList, limits api.ResourceList) api.Container { + return api.Container{ + Name: name, + Resources: getResourceRequirements(requests, limits), + } +} + +func newPod(name string, containers []api.Container) *api.Pod { + return &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + }, + Spec: api.PodSpec{ + Containers: containers, + }, + } +} + +func TestGetPodQOS(t *testing.T) { + testCases := []struct { + pod *api.Pod + expected api.PodQOSClass + }{ + { + pod: newPod("guaranteed", []api.Container{ + newContainer("guaranteed", getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")), + }), + expected: api.PodQOSGuaranteed, + }, + { + pod: newPod("best-effort", []api.Container{ + newContainer("best-effort", getResourceList("", ""), getResourceList("", "")), + }), + expected: api.PodQOSBestEffort, + }, + { + pod: newPod("burstable", []api.Container{ + newContainer("burstable", getResourceList("100m", "100Mi"), getResourceList("", "")), + }), + expected: api.PodQOSBurstable, + }, + } + for id, testCase := range testCases { + Strategy.PrepareForCreate(genericapirequest.NewContext(), testCase.pod) + actual := testCase.pod.Status.QOSClass + if actual != testCase.expected { + t.Errorf("[%d]: invalid qos pod %s, expected: %s, actual: %s", id, testCase.pod.Name, testCase.expected, actual) + } + } +} + func TestCheckGracefulDelete(t *testing.T) { defaultGracePeriod := int64(30) tcs := []struct { diff --git a/test/e2e/pods.go b/test/e2e/pods.go index d6ad33cbf62..db65f0db18e 100644 --- a/test/e2e/pods.go +++ b/test/e2e/pods.go @@ -24,6 +24,7 @@ import ( "strconv" "time" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/v1" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/labels" @@ -36,155 +37,203 @@ import ( . "github.com/onsi/gomega" ) -var _ = framework.KubeDescribe("Pods Delete Grace Period", func() { +var _ = framework.KubeDescribe("Pods Extended", func() { f := framework.NewDefaultFramework("pods") - var podClient *framework.PodClient - BeforeEach(func() { - podClient = f.PodClient() - }) - It("should be submitted and removed [Conformance]", func() { - By("creating the pod") - name := "pod-submit-remove-" + string(uuid.NewUUID()) - value := strconv.Itoa(time.Now().Nanosecond()) - pod := &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": "foo", - "time": value, - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - Image: "gcr.io/google_containers/nginx-slim:0.7", + + framework.KubeDescribe("Delete Grace Period", func() { + var podClient *framework.PodClient + BeforeEach(func() { + podClient = f.PodClient() + }) + It("should be submitted and removed [Conformance]", func() { + By("creating the pod") + name := "pod-submit-remove-" + string(uuid.NewUUID()) + value := strconv.Itoa(time.Now().Nanosecond()) + pod := &v1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": "foo", + "time": value, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "gcr.io/google_containers/nginx-slim:0.7", + }, }, }, - }, - } - - By("setting up watch") - selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) - options := v1.ListOptions{LabelSelector: selector.String()} - pods, err := podClient.List(options) - Expect(err).NotTo(HaveOccurred(), "failed to query for pod") - Expect(len(pods.Items)).To(Equal(0)) - options = v1.ListOptions{ - LabelSelector: selector.String(), - ResourceVersion: pods.ListMeta.ResourceVersion, - } - w, err := podClient.Watch(options) - Expect(err).NotTo(HaveOccurred(), "failed to set up watch") - - By("submitting the pod to kubernetes") - podClient.Create(pod) - - By("verifying the pod is in kubernetes") - selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) - options = v1.ListOptions{LabelSelector: selector.String()} - pods, err = podClient.List(options) - Expect(err).NotTo(HaveOccurred(), "failed to query for pod") - Expect(len(pods.Items)).To(Equal(1)) - - By("verifying pod creation was observed") - select { - case event, _ := <-w.ResultChan(): - if event.Type != watch.Added { - framework.Failf("Failed to observe pod creation: %v", event) } - case <-time.After(framework.PodStartTimeout): - Fail("Timeout while waiting for pod creation") - } - // We need to wait for the pod to be running, otherwise the deletion - // may be carried out immediately rather than gracefully. - framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) - // save the running pod - pod, err = podClient.Get(pod.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod") - - // start local proxy, so we can send graceful deletion over query string, rather than body parameter - cmd := framework.KubectlCmd("proxy", "-p", "0") - stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd) - Expect(err).NotTo(HaveOccurred(), "failed to start up proxy") - defer stdout.Close() - defer stderr.Close() - defer framework.TryKill(cmd) - buf := make([]byte, 128) - var n int - n, err = stdout.Read(buf) - Expect(err).NotTo(HaveOccurred(), "failed to read from kubectl proxy stdout") - output := string(buf[:n]) - proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)") - match := proxyRegexp.FindStringSubmatch(output) - Expect(len(match)).To(Equal(2)) - port, err := strconv.Atoi(match[1]) - Expect(err).NotTo(HaveOccurred(), "failed to convert port into string") - - endpoint := fmt.Sprintf("http://localhost:%d/api/v1/namespaces/%s/pods/%s?gracePeriodSeconds=30", port, pod.Namespace, pod.Name) - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - req, err := http.NewRequest("DELETE", endpoint, nil) - Expect(err).NotTo(HaveOccurred(), "failed to create http request") - - By("deleting the pod gracefully") - rsp, err := client.Do(req) - Expect(err).NotTo(HaveOccurred(), "failed to use http client to send delete") - - defer rsp.Body.Close() - - By("verifying the kubelet observed the termination notice") - Expect(wait.Poll(time.Second*5, time.Second*30, func() (bool, error) { - podList, err := framework.GetKubeletPods(f.ClientSet, pod.Spec.NodeName) - if err != nil { - framework.Logf("Unable to retrieve kubelet pods for node %v: %v", pod.Spec.NodeName, err) - return false, nil + By("setting up watch") + selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) + options := v1.ListOptions{LabelSelector: selector.String()} + pods, err := podClient.List(options) + Expect(err).NotTo(HaveOccurred(), "failed to query for pod") + Expect(len(pods.Items)).To(Equal(0)) + options = v1.ListOptions{ + LabelSelector: selector.String(), + ResourceVersion: pods.ListMeta.ResourceVersion, } - for _, kubeletPod := range podList.Items { - if pod.Name != kubeletPod.Name { - continue - } - if kubeletPod.ObjectMeta.DeletionTimestamp == nil { - framework.Logf("deletion has not yet been observed") - return false, nil - } - return true, nil - } - framework.Logf("no pod exists with the name we were looking for, assuming the termination request was observed and completed") - return true, nil - })).NotTo(HaveOccurred(), "kubelet never observed the termination notice") + w, err := podClient.Watch(options) + Expect(err).NotTo(HaveOccurred(), "failed to set up watch") - By("verifying pod deletion was observed") - deleted := false - timeout := false - var lastPod *v1.Pod - timer := time.After(30 * time.Second) - for !deleted && !timeout { + By("submitting the pod to kubernetes") + podClient.Create(pod) + + By("verifying the pod is in kubernetes") + selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) + options = v1.ListOptions{LabelSelector: selector.String()} + pods, err = podClient.List(options) + Expect(err).NotTo(HaveOccurred(), "failed to query for pod") + Expect(len(pods.Items)).To(Equal(1)) + + By("verifying pod creation was observed") select { case event, _ := <-w.ResultChan(): - if event.Type == watch.Deleted { - lastPod = event.Object.(*v1.Pod) - deleted = true + if event.Type != watch.Added { + framework.Failf("Failed to observe pod creation: %v", event) } - case <-timer: - timeout = true + case <-time.After(framework.PodStartTimeout): + Fail("Timeout while waiting for pod creation") } - } - if !deleted { - Fail("Failed to observe pod deletion") - } - Expect(lastPod.DeletionTimestamp).ToNot(BeNil()) - Expect(lastPod.Spec.TerminationGracePeriodSeconds).ToNot(BeZero()) + // We need to wait for the pod to be running, otherwise the deletion + // may be carried out immediately rather than gracefully. + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + // save the running pod + pod, err = podClient.Get(pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod") - selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) - options = v1.ListOptions{LabelSelector: selector.String()} - pods, err = podClient.List(options) - Expect(err).NotTo(HaveOccurred(), "failed to query for pods") - Expect(len(pods.Items)).To(Equal(0)) + // start local proxy, so we can send graceful deletion over query string, rather than body parameter + cmd := framework.KubectlCmd("proxy", "-p", "0") + stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to start up proxy") + defer stdout.Close() + defer stderr.Close() + defer framework.TryKill(cmd) + buf := make([]byte, 128) + var n int + n, err = stdout.Read(buf) + Expect(err).NotTo(HaveOccurred(), "failed to read from kubectl proxy stdout") + output := string(buf[:n]) + proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)") + match := proxyRegexp.FindStringSubmatch(output) + Expect(len(match)).To(Equal(2)) + port, err := strconv.Atoi(match[1]) + Expect(err).NotTo(HaveOccurred(), "failed to convert port into string") + endpoint := fmt.Sprintf("http://localhost:%d/api/v1/namespaces/%s/pods/%s?gracePeriodSeconds=30", port, pod.Namespace, pod.Name) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + req, err := http.NewRequest("DELETE", endpoint, nil) + Expect(err).NotTo(HaveOccurred(), "failed to create http request") + + By("deleting the pod gracefully") + rsp, err := client.Do(req) + Expect(err).NotTo(HaveOccurred(), "failed to use http client to send delete") + + defer rsp.Body.Close() + + By("verifying the kubelet observed the termination notice") + Expect(wait.Poll(time.Second*5, time.Second*30, func() (bool, error) { + podList, err := framework.GetKubeletPods(f.ClientSet, pod.Spec.NodeName) + if err != nil { + framework.Logf("Unable to retrieve kubelet pods for node %v: %v", pod.Spec.NodeName, err) + return false, nil + } + for _, kubeletPod := range podList.Items { + if pod.Name != kubeletPod.Name { + continue + } + if kubeletPod.ObjectMeta.DeletionTimestamp == nil { + framework.Logf("deletion has not yet been observed") + return false, nil + } + return true, nil + } + framework.Logf("no pod exists with the name we were looking for, assuming the termination request was observed and completed") + return true, nil + })).NotTo(HaveOccurred(), "kubelet never observed the termination notice") + + By("verifying pod deletion was observed") + deleted := false + timeout := false + var lastPod *v1.Pod + timer := time.After(30 * time.Second) + for !deleted && !timeout { + select { + case event, _ := <-w.ResultChan(): + if event.Type == watch.Deleted { + lastPod = event.Object.(*v1.Pod) + deleted = true + } + case <-timer: + timeout = true + } + } + if !deleted { + Fail("Failed to observe pod deletion") + } + + Expect(lastPod.DeletionTimestamp).ToNot(BeNil()) + Expect(lastPod.Spec.TerminationGracePeriodSeconds).ToNot(BeZero()) + + selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) + options = v1.ListOptions{LabelSelector: selector.String()} + pods, err = podClient.List(options) + Expect(err).NotTo(HaveOccurred(), "failed to query for pods") + Expect(len(pods.Items)).To(Equal(0)) + + }) + }) + + framework.KubeDescribe("Pods Set QOS Class", func() { + var podClient *framework.PodClient + BeforeEach(func() { + podClient = f.PodClient() + }) + It("should be submitted and removed [Conformance]", func() { + By("creating the pod") + name := "pod-qos-class-" + string(uuid.NewUUID()) + pod := &v1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": name, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "gcr.io/google_containers/nginx-slim:0.7", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("100Mi"), + }, + Requests: v1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("100Mi"), + }, + }, + }, + }, + }, + } + + By("submitting the pod to kubernetes") + podClient.Create(pod) + + By("verifying QOS class is set on the pod") + pod, err := podClient.Get(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "failed to query for pod") + Expect(pod.Status.QOSClass == v1.PodQOSGuaranteed) + }) }) }) diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index cb8ef1458b6..b4cc1569ef6 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -86,7 +86,6 @@ go_test( "//pkg/kubelet/dockertools:go_default_library", "//pkg/kubelet/images:go_default_library", "//pkg/kubelet/metrics:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/labels:go_default_library", "//pkg/metrics:go_default_library", "//pkg/runtime:go_default_library",