e2e: node: bootstrap podresources tests

Start e2e tests for the existing List() API.

Signed-off-by: Francesco Romani <fromani@redhat.com>

test/e2e_node/cpu_manager_test.go

@@ -81,14 +81,18 @@ func makeCPUManagerPod(podName string, ctnAttributes []ctnAttribute) *v1.Pod {
 	}
 }
 
-func deletePods(f *framework.Framework, podNames []string) {
+func deletePodSyncByName(f *framework.Framework, podName string) {
-	for _, podName := range podNames {
 	gp := int64(0)
 	delOpts := metav1.DeleteOptions{
 		GracePeriodSeconds: &gp,
 	}
 	f.PodClient().DeleteSync(podName, delOpts, framework.DefaultPodDeletionTimeout)
 }
+
+func deletePods(f *framework.Framework, podNames []string) {
+	for _, podName := range podNames {
+		deletePodSyncByName(f, podName)
+	}
 }
 
 func getLocalNodeCPUDetails(f *framework.Framework) (cpuCapVal int64, cpuAllocVal int64, cpuResVal int64) {

test/e2e_node/podresources_test.go

@@ -0,0 +1,289 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2enode
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
+	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
+	"k8s.io/kubernetes/pkg/kubelet/util"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+func makePodResourcesTestPod(podName, cntName, devName, devCount string) *v1.Pod {
+	cnt := v1.Container{
+		Name:  cntName,
+		Image: busyboxImage,
+		Resources: v1.ResourceRequirements{
+			Requests: v1.ResourceList{},
+			Limits:   v1.ResourceList{},
+		},
+		Command: []string{"sh", "-c", "sleep 1d"},
+	}
+	if devName != "" && devCount != "" {
+		cnt.Resources.Requests[v1.ResourceName(devName)] = resource.MustParse(devCount)
+		cnt.Resources.Limits[v1.ResourceName(devName)] = resource.MustParse(devCount)
+	}
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+		},
+		Spec: v1.PodSpec{
+			RestartPolicy: v1.RestartPolicyNever,
+			Containers: []v1.Container{
+				cnt,
+			},
+		},
+	}
+}
+
+func countPodResources(podIdx int, pr *kubeletpodresourcesv1.PodResources) int {
+	ns := pr.GetNamespace()
+	devCount := 0
+	for cntIdx, cnt := range pr.GetContainers() {
+		if len(cnt.Devices) > 0 {
+			for devIdx, dev := range cnt.Devices {
+				framework.Logf("#%02d/%02d/%02d - %s/%s/%s   %s -> %s", podIdx, cntIdx, devIdx, ns, pr.GetName(), cnt.Name, dev.ResourceName, strings.Join(dev.DeviceIds, ", "))
+				devCount++
+			}
+		} else {
+			framework.Logf("#%02d/%02d/%02d - %s/%s/%s   No resources", podIdx, cntIdx, 0, ns, pr.GetName(), cnt.Name)
+		}
+	}
+	return devCount
+}
+
+func getPodResources(cli kubeletpodresourcesv1.PodResourcesListerClient) ([]*kubeletpodresourcesv1.PodResources, []*kubeletpodresourcesv1.PodResources) {
+	resp, err := cli.List(context.TODO(), &kubeletpodresourcesv1.ListPodResourcesRequest{})
+	framework.ExpectNoError(err)
+
+	res := []*kubeletpodresourcesv1.PodResources{}
+	noRes := []*kubeletpodresourcesv1.PodResources{}
+	for idx, podResource := range resp.GetPodResources() {
+		if countPodResources(idx, podResource) > 0 {
+			res = append(res, podResource)
+		} else {
+			noRes = append(noRes, podResource)
+		}
+	}
+	return res, noRes
+}
+
+type podDesc struct {
+	podName        string
+	resourceName   string
+	resourceAmount string
+}
+
+type testPodData struct {
+	PodMap map[string]*v1.Pod
+}
+
+func newTestPodData() *testPodData {
+	return &testPodData{
+		PodMap: make(map[string]*v1.Pod),
+	}
+}
+
+func (tpd *testPodData) createPodsForTest(f *framework.Framework, podReqs []podDesc) {
+	for _, podReq := range podReqs {
+		pod := makePodResourcesTestPod(podReq.podName, "cnt-0", podReq.resourceName, podReq.resourceAmount)
+		pod = f.PodClient().CreateSync(pod)
+
+		framework.Logf("created pod %s", podReq.podName)
+		tpd.PodMap[podReq.podName] = pod
+	}
+}
+
+func (tpd *testPodData) deletePodsForTest(f *framework.Framework) {
+	for podName := range tpd.PodMap {
+		deletePodSyncByName(f, podName)
+	}
+}
+
+func (tpd *testPodData) deletePod(f *framework.Framework, podName string) {
+	_, ok := tpd.PodMap[podName]
+	if !ok {
+		return
+	}
+	deletePodSyncByName(f, podName)
+	delete(tpd.PodMap, podName)
+}
+
+func expectPodResources(cli kubeletpodresourcesv1.PodResourcesListerClient, expectedPodsWithResources, expectedPodsWithoutResources int) {
+	gomega.EventuallyWithOffset(1, func() error {
+		podResources, noResources := getPodResources(cli)
+		if len(podResources) != expectedPodsWithResources {
+			return fmt.Errorf("pod with resources: expected %d found %d", expectedPodsWithResources, len(podResources))
+		}
+		if len(noResources) != expectedPodsWithoutResources {
+			return fmt.Errorf("pod WITHOUT resources: expected %d found %d", expectedPodsWithoutResources, len(noResources))
+		}
+		return nil
+	}, time.Minute, 10*time.Second).Should(gomega.BeNil())
+}
+
+func podresourcesListTests(f *framework.Framework, cli kubeletpodresourcesv1.PodResourcesListerClient, sd *sriovData) {
+	var podResources []*kubeletpodresourcesv1.PodResources
+	var noResources []*kubeletpodresourcesv1.PodResources
+	var tpd *testPodData
+
+	ginkgo.By("checking the output when no pods are present")
+	expectPodResources(cli, 0, 1) // sriovdp
+
+	tpd = newTestPodData()
+	ginkgo.By("checking the output when only pods which don't require resources are present")
+	tpd.createPodsForTest(f, []podDesc{
+		{
+			podName: "pod-00",
+		},
+		{
+			podName: "pod-01",
+		},
+	})
+	expectPodResources(cli, 0, 2+1) // test pods + sriovdp
+	tpd.deletePodsForTest(f)
+
+	tpd = newTestPodData()
+	ginkgo.By("checking the output when only a subset of pods require resources")
+	tpd.createPodsForTest(f, []podDesc{
+		{
+			podName: "pod-00",
+		},
+		{
+			podName:        "pod-01",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+		{
+			podName: "pod-02",
+		},
+		{
+			podName:        "pod-03",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+	})
+	expectPodResources(cli, 2, 2+1) // test pods + sriovdp
+	// TODO check for specific pods
+	tpd.deletePodsForTest(f)
+
+	tpd = newTestPodData()
+	ginkgo.By("checking the output when creating pods which require resources between calls")
+	tpd.createPodsForTest(f, []podDesc{
+		{
+			podName: "pod-00",
+		},
+		{
+			podName:        "pod-01",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+		{
+			podName: "pod-02",
+		},
+	})
+	podResources, noResources = getPodResources(cli)
+	framework.ExpectEqual(len(podResources), 1)
+	framework.ExpectEqual(len(noResources), 2+1) // test pods + sriovdp
+	// TODO check for specific pods
+
+	tpd.createPodsForTest(f, []podDesc{
+		{
+			podName:        "pod-03",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+	})
+	podResources, noResources = getPodResources(cli)
+	framework.ExpectEqual(len(podResources), 2)
+	framework.ExpectEqual(len(noResources), 2+1) // test pods + sriovdp
+	// TODO check for specific pods
+	tpd.deletePodsForTest(f)
+
+	tpd = newTestPodData()
+	ginkgo.By("checking the output when deleting pods which require resources between calls")
+	tpd.createPodsForTest(f, []podDesc{
+		{
+			podName: "pod-00",
+		},
+		{
+			podName:        "pod-01",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+		{
+			podName: "pod-02",
+		},
+		{
+			podName:        "pod-03",
+			resourceName:   sd.resourceName,
+			resourceAmount: "1",
+		},
+	})
+	podResources, noResources = getPodResources(cli)
+	framework.ExpectEqual(len(podResources), 2)
+	framework.ExpectEqual(len(noResources), 2+1) // test pods + sriovdp
+	// TODO check for specific pods
+
+	tpd.deletePod(f, "pod-01")
+	podResources, noResources = getPodResources(cli)
+	framework.ExpectEqual(len(podResources), 1)
+	framework.ExpectEqual(len(noResources), 2+1) // test pods + sriovdp
+	// TODO check for specific pods
+	tpd.deletePodsForTest(f)
+}
+
+// Serial because the test updates kubelet configuration.
+var _ = SIGDescribe("POD Resources [Serial] [Feature:PODResources][NodeFeature:PODResources]", func() {
+	f := framework.NewDefaultFramework("podresources-test")
+
+	ginkgo.Context("With SRIOV devices in the system", func() {
+		ginkgo.It("should return the expected responses from List()", func() {
+			// this is a very rough check. We just want to rule out system that does NOT have any SRIOV device.
+			if sriovdevCount, err := countSRIOVDevices(); err != nil || sriovdevCount == 0 {
+				e2eskipper.Skipf("this test is meant to run on a system with at least one configured VF from SRIOV device")
+			}
+
+			configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
+			sd := setupSRIOVConfigOrFail(f, configMap)
+			defer teardownSRIOVConfigOrFail(f, sd)
+
+			waitForSRIOVResources(f, sd)
+
+			endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
+			framework.ExpectNoError(err)
+
+			cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
+			defer conn.Close()
+
+			podresourcesListTests(f, cli, sd)
+		})
+	})
+})

test/e2e_node/topology_manager_test.go

@@ -89,13 +89,17 @@ func detectCoresPerSocket() int {
 	return coreCount
 }
 
-func detectSRIOVDevices() int {
+func countSRIOVDevices() (int, error) {
 	outData, err := exec.Command("/bin/sh", "-c", "ls /sys/bus/pci/devices/*/physfn | wc -w").Output()
-	framework.ExpectNoError(err)
+	if err != nil {
+		return -1, err
+	}
+	return strconv.Atoi(strings.TrimSpace(string(outData)))
+}
 
-	devCount, err := strconv.Atoi(strings.TrimSpace(string(outData)))
+func detectSRIOVDevices() int {
+	devCount, err := countSRIOVDevices()
 	framework.ExpectNoError(err)
-
 	return devCount
 }
 
@@ -434,7 +438,7 @@ func runTopologyManagerPositiveTest(f *framework.Framework, numPods int, ctnAttr
 		pod := pods[podID]
 		framework.Logf("deleting the pod %s/%s and waiting for container removal",
 			pod.Namespace, pod.Name)
-		deletePods(f, []string{pod.Name})
+		deletePodSyncByName(f, pod.Name)
 		waitForAllContainerRemoval(pod.Name, pod.Namespace)
 	}
 }
@@ -462,7 +466,7 @@ func runTopologyManagerNegativeTest(f *framework.Framework, ctnAttrs, initCtnAtt
 		framework.Failf("pod %s failed for wrong reason: %q", pod.Name, pod.Status.Reason)
 	}
 
-	deletePods(f, []string{pod.Name})
+	deletePodSyncByName(f, pod.Name)
 }
 
 func isTopologyAffinityError(pod *v1.Pod) bool {