mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Move pod level hugepage test from e2e to e2e node
This commit is contained in:
Kevin Torres
parent
98e362c049
commit
7b38bff6ec
@ -29,17 +29,12 @@ import (
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
v1resource "k8s.io/kubernetes/pkg/api/v1/resource"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
kubecm "k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
|
||||
utils "k8s.io/kubernetes/test/utils"
|
||||
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
)
|
||||
@ -48,14 +43,9 @@ const (
|
||||
cgroupv2CPUWeight string = "cpu.weight"
|
||||
cgroupv2CPULimit string = "cpu.max"
|
||||
cgroupv2MemLimit string = "memory.max"
|
||||
|
||||
cgroupv2HugeTLBPrefix string = "hugetlb"
|
||||
cgroupv2HugeTLBRsvd string = "rsvd"
|
||||
|
||||
cgroupFsPath string = "/sys/fs/cgroup"
|
||||
mountPath string = "/sysfscgroup"
|
||||
|
||||
CPUPeriod string = "100000"
|
||||
cgroupFsPath string = "/sys/fs/cgroup"
|
||||
CPUPeriod string = "100000"
|
||||
mountPath string = "/sysfscgroup"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -79,7 +69,6 @@ var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLe
|
||||
e2eskipper.Skipf("not supported on cgroupv1 -- skipping")
|
||||
}
|
||||
})
|
||||
|
||||
podLevelResourcesTests(f)
|
||||
})
|
||||
|
||||
@ -115,7 +104,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool {
|
||||
|
||||
func makeObjectMetadata(name, namespace string) metav1.ObjectMeta {
|
||||
return metav1.ObjectMeta{
|
||||
Name: name, Namespace: namespace,
|
||||
Name: "testpod", Namespace: namespace,
|
||||
Labels: map[string]string{"time": strconv.Itoa(time.Now().Nanosecond())},
|
||||
}
|
||||
}
|
||||
@ -124,16 +113,11 @@ type containerInfo struct {
|
||||
Name string
|
||||
Resources *resourceInfo
|
||||
}
|
||||
|
||||
type resourceInfo struct {
|
||||
CPUReq string
|
||||
CPULim string
|
||||
MemReq string
|
||||
MemLim string
|
||||
HugePagesReq2Mi string
|
||||
HugePagesLim2Mi string
|
||||
HugePagesReq1Gi string
|
||||
HugePagesLim1Gi string
|
||||
CPUReq string
|
||||
CPULim string
|
||||
MemReq string
|
||||
MemLim string
|
||||
}
|
||||
|
||||
func makeContainer(info containerInfo) v1.Container {
|
||||
@ -156,7 +140,7 @@ func makeContainer(info containerInfo) v1.Container {
|
||||
func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
|
||||
var res v1.ResourceRequirements
|
||||
if info != nil {
|
||||
if info.CPUReq != "" || info.MemReq != "" || info.HugePagesReq2Mi != "" || info.HugePagesReq1Gi != "" {
|
||||
if info.CPUReq != "" || info.MemReq != "" {
|
||||
res.Requests = make(v1.ResourceList)
|
||||
}
|
||||
if info.CPUReq != "" {
|
||||
@ -165,14 +149,8 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
|
||||
if info.MemReq != "" {
|
||||
res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq)
|
||||
}
|
||||
if info.HugePagesReq2Mi != "" {
|
||||
res.Requests[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesReq2Mi)
|
||||
}
|
||||
if info.HugePagesReq1Gi != "" {
|
||||
res.Requests[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesReq1Gi)
|
||||
}
|
||||
|
||||
if info.CPULim != "" || info.MemLim != "" || info.HugePagesLim2Mi != "" || info.HugePagesLim1Gi != "" {
|
||||
if info.CPULim != "" || info.MemLim != "" {
|
||||
res.Limits = make(v1.ResourceList)
|
||||
}
|
||||
if info.CPULim != "" {
|
||||
@ -181,12 +159,6 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
|
||||
if info.MemLim != "" {
|
||||
res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim)
|
||||
}
|
||||
if info.HugePagesLim2Mi != "" {
|
||||
res.Limits[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesLim2Mi)
|
||||
}
|
||||
if info.HugePagesLim1Gi != "" {
|
||||
res.Limits[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesLim1Gi)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
@ -239,7 +211,7 @@ func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) {
|
||||
}
|
||||
|
||||
// TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils.
|
||||
func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) error {
|
||||
func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error {
|
||||
ginkgo.GinkgoHelper()
|
||||
cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_"))
|
||||
framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd)
|
||||
@ -275,70 +247,6 @@ func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) e
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
|
||||
}
|
||||
|
||||
// Verify cgroup limits for all the hugepage sizes in the pod
|
||||
for resourceName, resourceAmount := range expectedResources.Limits {
|
||||
if !v1resource.IsHugePageResourceName(resourceName) {
|
||||
continue
|
||||
}
|
||||
|
||||
pageSize, err := v1helper.HugePageSizeFromResourceName(resourceName)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage size: %w", err))
|
||||
}
|
||||
|
||||
sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value())
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage unit size: %w", err))
|
||||
}
|
||||
|
||||
hugepageCgroupv2Limits := []string{
|
||||
fmt.Sprintf("%s.%s.max", cgroupv2HugeTLBPrefix, sizeString),
|
||||
fmt.Sprintf("%s.%s.%s.max", cgroupv2HugeTLBPrefix, sizeString, cgroupv2HugeTLBRsvd),
|
||||
}
|
||||
expectedHugepageLim := strconv.FormatInt(resourceAmount.Value(), 10)
|
||||
|
||||
for _, hugepageCgroupv2Limit := range hugepageCgroupv2Limits {
|
||||
hugepageLimCgPath := fmt.Sprintf("%s/%s", podCgPath, hugepageCgroupv2Limit)
|
||||
err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, hugepageLimCgPath, expectedHugepageLim)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify hugepage limit cgroup value: %w, path: %s", err, hugepageLimCgPath))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return utilerrors.NewAggregate(errs)
|
||||
}
|
||||
|
||||
func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
|
||||
var errs []error
|
||||
for _, container := range pod.Spec.Containers {
|
||||
if pod.Spec.Resources == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if pod.Spec.Resources.Limits.Memory() != nil && container.Resources.Limits.Memory() == nil {
|
||||
expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10)
|
||||
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if pod.Spec.Resources.Limits.Cpu() != nil && container.Resources.Limits.Cpu() == nil {
|
||||
cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod)
|
||||
expectedCPULimit := strconv.FormatInt(cpuQuota, 10)
|
||||
expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod)
|
||||
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(KevinTMtz) - Check for all hugepages for the pod, for this is
|
||||
// required to enabled the Containerd Cgroup value, because if not, HugeTLB
|
||||
// cgroup values will be just set to max
|
||||
}
|
||||
return utilerrors.NewAggregate(errs)
|
||||
}
|
||||
|
||||
@ -349,7 +257,7 @@ func podLevelResourcesTests(f *framework.Framework) {
|
||||
// and limits for the pod. If pod-level resource specifications
|
||||
// are specified, totalPodResources is equal to pod-level resources.
|
||||
// Otherwise, it is calculated by aggregating resource requests and
|
||||
// limits from all containers within the pod.
|
||||
// limits from all containers within the pod..
|
||||
totalPodResources *resourceInfo
|
||||
}
|
||||
|
||||
@ -358,7 +266,6 @@ func podLevelResourcesTests(f *framework.Framework) {
|
||||
podResources *resourceInfo
|
||||
containers []containerInfo
|
||||
expected expectedPodConfig
|
||||
hugepages map[string]int
|
||||
}
|
||||
|
||||
tests := []testCase{
|
||||
@ -442,108 +349,10 @@ func podLevelResourcesTests(f *framework.Framework) {
|
||||
totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Guaranteed QoS pod hugepages, no container resources, single page size",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
|
||||
containers: []containerInfo{{Name: "c1"}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSGuaranteed,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Burstable QoS pod hugepages, container resources, single page size",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
|
||||
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSBurstable,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Burstable QoS pod hugepages, container resources, single page size, pod level does not specify hugepages",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
|
||||
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSBurstable,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Guaranteed QoS pod hugepages, no container resources, multiple page size",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"},
|
||||
containers: []containerInfo{{Name: "c1"}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSGuaranteed,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 5,
|
||||
v1.ResourceHugePagesPrefix + "1Gi": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Burstable QoS pod hugepages, container resources, multiple page size",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"},
|
||||
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSBurstable,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 5,
|
||||
v1.ResourceHugePagesPrefix + "1Gi": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Burstable QoS pod hugepages, container resources, multiple page size, pod level does not specify hugepages",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
|
||||
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSBurstable,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 2,
|
||||
v1.ResourceHugePagesPrefix + "1Gi": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Burstable QoS pod hugepages, container resources, different page size between pod and container level",
|
||||
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
|
||||
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
|
||||
expected: expectedPodConfig{
|
||||
qos: v1.PodQOSBurstable,
|
||||
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
|
||||
},
|
||||
hugepages: map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": 5,
|
||||
v1.ResourceHugePagesPrefix + "1Gi": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
ginkgo.It(tc.name, func(ctx context.Context) {
|
||||
// Pre-allocate hugepages in the node
|
||||
if tc.hugepages != nil {
|
||||
utils.SetHugepages(ctx, tc.hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
utils.RestartKubelet(ctx, false)
|
||||
|
||||
utils.WaitForHugepages(ctx, f, tc.hugepages)
|
||||
}
|
||||
|
||||
podMetadata := makeObjectMetadata("testpod", f.Namespace.Name)
|
||||
testPod := makePod(&podMetadata, tc.podResources, tc.containers)
|
||||
|
||||
@ -558,7 +367,7 @@ func podLevelResourcesTests(f *framework.Framework) {
|
||||
verifyQoS(*pod, tc.expected.qos)
|
||||
|
||||
ginkgo.By("verifying pod cgroup values")
|
||||
err := verifyPodCgroups(f, pod, tc.expected.totalPodResources)
|
||||
err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources)
|
||||
framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err)
|
||||
|
||||
ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits")
|
||||
@ -568,16 +377,32 @@ func podLevelResourcesTests(f *framework.Framework) {
|
||||
ginkgo.By("deleting pods")
|
||||
delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
|
||||
framework.ExpectNoError(delErr, "failed to delete pod %s", delErr)
|
||||
|
||||
// Release pre-allocated hugepages
|
||||
if tc.hugepages != nil {
|
||||
utils.ReleaseHugepages(ctx, tc.hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
utils.RestartKubelet(ctx, true)
|
||||
|
||||
utils.WaitForHugepages(ctx, f, tc.hugepages)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
|
||||
var errs []error
|
||||
for _, container := range pod.Spec.Containers {
|
||||
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil &&
|
||||
container.Resources.Limits.Memory() == nil {
|
||||
expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10)
|
||||
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil &&
|
||||
container.Resources.Limits.Cpu() == nil {
|
||||
cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod)
|
||||
expectedCPULimit := strconv.FormatInt(cpuQuota, 10)
|
||||
expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod)
|
||||
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
return utilerrors.NewAggregate(errs)
|
||||
}
|
||||
|
||||
@ -19,6 +19,10 @@ package e2enode
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
@ -29,11 +33,12 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
testutils "k8s.io/kubernetes/test/utils"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
)
|
||||
|
||||
@ -115,8 +120,66 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity,
|
||||
return pod
|
||||
}
|
||||
|
||||
func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
// configureHugePages attempts to allocate hugepages of the specified size
|
||||
func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error {
|
||||
// Compact memory to make bigger contiguous blocks of memory available
|
||||
// before allocating huge pages.
|
||||
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
|
||||
if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil {
|
||||
if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// e.g. hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile)
|
||||
|
||||
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix)
|
||||
if numaNodeID != nil {
|
||||
// e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix)
|
||||
}
|
||||
|
||||
// Reserve number of hugepages
|
||||
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
|
||||
command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile)
|
||||
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// verify that the number of hugepages was updated
|
||||
// e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
|
||||
command = fmt.Sprintf("cat %s", hugepagesFile)
|
||||
outData, err := exec.Command("/bin/sh", "-c", command).Output()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
framework.Logf("Hugepages total is set to %v", numHugePages)
|
||||
if numHugePages == hugepagesCount {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages)
|
||||
}
|
||||
|
||||
// isHugePageAvailable returns true if hugepages of the specified size is available on the host
|
||||
func isHugePageAvailable(hugepagesSize int) bool {
|
||||
path := fmt.Sprintf("%s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod {
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "hugepages-",
|
||||
Namespace: f.Namespace.Name,
|
||||
@ -124,18 +187,110 @@ func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "container" + string(uuid.NewUUID()),
|
||||
Image: busyboxImage,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: limits,
|
||||
},
|
||||
Name: "container" + string(uuid.NewUUID()),
|
||||
Image: busyboxImage,
|
||||
Command: []string{"sleep", "3600"},
|
||||
VolumeMounts: mounts,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: containerLimits,
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: volumes,
|
||||
},
|
||||
}
|
||||
|
||||
if podLimits != nil {
|
||||
pod.Spec.Resources = &v1.ResourceRequirements{
|
||||
Limits: podLimits,
|
||||
}
|
||||
}
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
func setHugepages(ctx context.Context, hugepages map[string]int) {
|
||||
for hugepagesResource, count := range hugepages {
|
||||
size := resourceToSize[hugepagesResource]
|
||||
ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size))
|
||||
if !isHugePageAvailable(size) {
|
||||
e2eskipper.Skipf("skipping test because hugepages of size %d not supported", size)
|
||||
return
|
||||
}
|
||||
|
||||
ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size))
|
||||
gomega.Eventually(ctx, func() error {
|
||||
if err := configureHugePages(size, count, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, 30*time.Second, framework.Poll).Should(gomega.Succeed())
|
||||
}
|
||||
}
|
||||
|
||||
func waitForHugepages(f *framework.Framework, ctx context.Context, hugepages map[string]int) {
|
||||
ginkgo.By("Waiting for hugepages resource to become available on the local node")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) error {
|
||||
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for hugepagesResource, count := range hugepages {
|
||||
capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)]
|
||||
if !ok {
|
||||
return fmt.Errorf("the node does not have the resource %s", hugepagesResource)
|
||||
}
|
||||
|
||||
size, succeed := capacity.AsInt64()
|
||||
if !succeed {
|
||||
return fmt.Errorf("failed to convert quantity to int64")
|
||||
}
|
||||
|
||||
expectedSize := count * resourceToSize[hugepagesResource] * 1024
|
||||
if size != int64(expectedSize) {
|
||||
return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, time.Minute, framework.Poll).Should(gomega.Succeed())
|
||||
}
|
||||
|
||||
func releaseHugepages(ctx context.Context, hugepages map[string]int) {
|
||||
ginkgo.By("Releasing hugepages")
|
||||
gomega.Eventually(ctx, func() error {
|
||||
for hugepagesResource := range hugepages {
|
||||
command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile)
|
||||
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, 30*time.Second, framework.Poll).Should(gomega.Succeed())
|
||||
}
|
||||
|
||||
func runHugePagesTests(f *framework.Framework, ctx context.Context, testpod *v1.Pod, expectedHugepageLimits v1.ResourceList, mounts []v1.VolumeMount, hugepages map[string]int) {
|
||||
ginkgo.By("getting mounts for the test pod")
|
||||
command := []string{"mount"}
|
||||
|
||||
out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...)
|
||||
|
||||
for _, mount := range mounts {
|
||||
ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath))
|
||||
gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath))
|
||||
}
|
||||
|
||||
for resourceName := range hugepages {
|
||||
verifyPod := makePodToVerifyHugePages(
|
||||
"pod"+string(testpod.UID),
|
||||
expectedHugepageLimits[v1.ResourceName(resourceName)],
|
||||
resourceToCgroup[resourceName],
|
||||
)
|
||||
ginkgo.By("checking if the expected hugetlb settings were applied")
|
||||
e2epod.NewPodClient(f).Create(ctx, verifyPod)
|
||||
err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name)
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Serial because the test updates kubelet configuration.
|
||||
@ -193,48 +348,24 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
|
||||
ginkgo.When("start the pod", func() {
|
||||
var (
|
||||
testpod *v1.Pod
|
||||
limits v1.ResourceList
|
||||
mounts []v1.VolumeMount
|
||||
volumes []v1.Volume
|
||||
hugepages map[string]int
|
||||
testpod *v1.Pod
|
||||
expectedHugepageLimits v1.ResourceList
|
||||
containerLimits v1.ResourceList
|
||||
mounts []v1.VolumeMount
|
||||
volumes []v1.Volume
|
||||
hugepages map[string]int
|
||||
)
|
||||
|
||||
runHugePagesTests := func() {
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
ginkgo.By("getting mounts for the test pod")
|
||||
command := []string{"mount"}
|
||||
out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...)
|
||||
|
||||
for _, mount := range mounts {
|
||||
ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath))
|
||||
gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath))
|
||||
}
|
||||
|
||||
for resourceName := range hugepages {
|
||||
verifyPod := makePodToVerifyHugePages(
|
||||
"pod"+string(testpod.UID),
|
||||
testpod.Spec.Containers[0].Resources.Limits[v1.ResourceName(resourceName)],
|
||||
resourceToCgroup[resourceName],
|
||||
)
|
||||
ginkgo.By("checking if the expected hugetlb settings were applied")
|
||||
e2epod.NewPodClient(f).Create(ctx, verifyPod)
|
||||
err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name)
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// setup
|
||||
ginkgo.JustBeforeEach(func(ctx context.Context) {
|
||||
testutils.SetHugepages(ctx, hugepages)
|
||||
setHugepages(ctx, hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
restartKubelet(ctx, true)
|
||||
|
||||
testutils.WaitForHugepages(ctx, f, hugepages)
|
||||
waitForHugepages(f, ctx, hugepages)
|
||||
|
||||
pod := getHugepagesTestPod(f, limits, mounts, volumes)
|
||||
pod := getHugepagesTestPod(f, nil, containerLimits, mounts, volumes)
|
||||
|
||||
ginkgo.By("by running a test pod that requests hugepages")
|
||||
testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
|
||||
@ -245,18 +376,21 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name))
|
||||
e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
|
||||
|
||||
testutils.ReleaseHugepages(ctx, hugepages)
|
||||
releaseHugepages(ctx, hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
restartKubelet(ctx, true)
|
||||
|
||||
testutils.WaitForHugepages(ctx, f, hugepages)
|
||||
waitForHugepages(f, ctx, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.Context("with the resources requests that contain only one hugepages resource ", func() {
|
||||
ginkgo.Context("with the backward compatible API", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
limits = v1.ResourceList{
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
@ -280,12 +414,17 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
hugepages = map[string]int{hugepagesResourceName2Mi: 5}
|
||||
})
|
||||
// run tests
|
||||
runHugePagesTests()
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("with the new API", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
limits = v1.ResourceList{
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
@ -309,7 +448,9 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
hugepages = map[string]int{hugepagesResourceName2Mi: 5}
|
||||
})
|
||||
|
||||
runHugePagesTests()
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
@ -323,7 +464,11 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
hugepagesResourceName2Mi: 5,
|
||||
hugepagesResourceName1Gi: 1,
|
||||
}
|
||||
limits = v1.ResourceList{
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
@ -359,7 +504,443 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
|
||||
}
|
||||
})
|
||||
|
||||
runHugePagesTests()
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
hugepagesResourceName1Gi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// Serial because the test updates kubelet configuration.
|
||||
var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, func() {
|
||||
f := framework.NewDefaultFramework("pod-level-hugepages-resources")
|
||||
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
|
||||
|
||||
ginkgo.When("pod level resources", func() {
|
||||
var (
|
||||
testpod *v1.Pod
|
||||
expectedHugepageLimits v1.ResourceList
|
||||
podLimits v1.ResourceList
|
||||
containerLimits v1.ResourceList
|
||||
mounts []v1.VolumeMount
|
||||
volumes []v1.Volume
|
||||
hugepages map[string]int
|
||||
)
|
||||
|
||||
// setup
|
||||
ginkgo.JustBeforeEach(func(ctx context.Context) {
|
||||
e2eskipper.SkipUnlessFeatureGateEnabled(features.PodLevelResources)
|
||||
|
||||
setHugepages(ctx, hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
restartKubelet(ctx, true)
|
||||
|
||||
waitForHugepages(f, ctx, hugepages)
|
||||
|
||||
pod := getHugepagesTestPod(f, podLimits, containerLimits, mounts, volumes)
|
||||
|
||||
ginkgo.By("by running a test pod that requests hugepages")
|
||||
|
||||
testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
|
||||
|
||||
framework.Logf("Test pod name: %s", testpod.Name)
|
||||
})
|
||||
|
||||
// we should use JustAfterEach because framework will teardown the client under the AfterEach method
|
||||
ginkgo.JustAfterEach(func(ctx context.Context) {
|
||||
ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name))
|
||||
e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
|
||||
|
||||
releaseHugepages(ctx, hugepages)
|
||||
|
||||
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
|
||||
restartKubelet(ctx, true)
|
||||
|
||||
waitForHugepages(f, ctx, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.Context("pod hugepages, no container hugepages, single page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("pod hugepages, container hugepages, single page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("no pod hugepages, container hugepages, single page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("pod hugepages, no container hugepages, multiple page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
hugepagesResourceName1Gi: 1,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
MountPath: "/hugepages-1Gi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages1Gi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
hugepagesResourceName1Gi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("pod hugepages, container hugepages, multiple page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
hugepagesResourceName1Gi: 1,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
MountPath: "/hugepages-1Gi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages1Gi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
hugepagesResourceName1Gi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("no pod hugepages, container hugepages, multiple page size", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
hugepagesResourceName1Gi: 1,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
MountPath: "/hugepages-1Gi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages1Gi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 0,
|
||||
hugepagesResourceName1Gi: 0,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("pod hugepages, container hugepages, different page size between pod and container level", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
hugepages = map[string]int{
|
||||
hugepagesResourceName2Mi: 5,
|
||||
hugepagesResourceName1Gi: 1,
|
||||
}
|
||||
expectedHugepageLimits = v1.ResourceList{
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
podLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
|
||||
}
|
||||
containerLimits = v1.ResourceList{
|
||||
v1.ResourceCPU: resource.MustParse("10m"),
|
||||
v1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
|
||||
}
|
||||
mounts = []v1.VolumeMount{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
MountPath: "/hugepages-2Mi",
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
MountPath: "/hugepages-1Gi",
|
||||
},
|
||||
}
|
||||
volumes = []v1.Volume{
|
||||
{
|
||||
Name: "hugepages-2mi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages2Mi,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "hugepages-1gi",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
Medium: mediumHugepages1Gi,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
|
||||
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
|
||||
})
|
||||
|
||||
ginkgo.JustAfterEach(func() {
|
||||
hugepages = map[string]int{
|
||||
|
||||
@ -42,7 +42,6 @@ import (
|
||||
"k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
testutils "k8s.io/kubernetes/test/utils"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
"k8s.io/utils/cpuset"
|
||||
"k8s.io/utils/pointer"
|
||||
@ -315,7 +314,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
|
||||
}
|
||||
|
||||
if is2MiHugepagesSupported == nil {
|
||||
is2MiHugepagesSupported = pointer.BoolPtr(testutils.IsHugePageAvailable(hugepagesSize2M))
|
||||
is2MiHugepagesSupported = pointer.BoolPtr(isHugePageAvailable(hugepagesSize2M))
|
||||
}
|
||||
|
||||
if len(allNUMANodes) == 0 {
|
||||
@ -326,7 +325,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
|
||||
if *is2MiHugepagesSupported {
|
||||
ginkgo.By("Configuring hugepages")
|
||||
gomega.Eventually(ctx, func() error {
|
||||
return testutils.ConfigureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0))
|
||||
return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0))
|
||||
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
|
||||
}
|
||||
})
|
||||
@ -359,7 +358,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
|
||||
ginkgo.By("Releasing allocated hugepages")
|
||||
gomega.Eventually(ctx, func() error {
|
||||
// configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes
|
||||
return testutils.ConfigureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0))
|
||||
return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0))
|
||||
}, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages")
|
||||
}
|
||||
})
|
||||
|
||||
@ -16,39 +16,7 @@ limitations under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
)
|
||||
|
||||
const (
|
||||
hugepagesCapacityFile = "nr_hugepages"
|
||||
hugepagesDirPrefix = "/sys/kernel/mm/hugepages/hugepages"
|
||||
|
||||
hugepagesSize2M = 2048
|
||||
hugepagesSize1G = 1048576
|
||||
)
|
||||
|
||||
var (
|
||||
resourceToSize = map[string]int{
|
||||
v1.ResourceHugePagesPrefix + "2Mi": hugepagesSize2M,
|
||||
v1.ResourceHugePagesPrefix + "1Gi": hugepagesSize1G,
|
||||
}
|
||||
)
|
||||
import v1 "k8s.io/api/core/v1"
|
||||
|
||||
// GetNodeCondition extracts the provided condition from the given status and returns that.
|
||||
// Returns nil and -1 if the condition is not present, and the index of the located condition.
|
||||
@ -63,158 +31,3 @@ func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType)
|
||||
}
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
func SetHugepages(ctx context.Context, hugepages map[string]int) {
|
||||
for hugepagesResource, count := range hugepages {
|
||||
size := resourceToSize[hugepagesResource]
|
||||
ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size))
|
||||
if !IsHugePageAvailable(size) {
|
||||
skipf("skipping test because hugepages of size %d not supported", size)
|
||||
return
|
||||
}
|
||||
|
||||
ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size))
|
||||
gomega.Eventually(ctx, func() error {
|
||||
if err := ConfigureHugePages(size, count, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
|
||||
}
|
||||
}
|
||||
|
||||
func IsHugePageAvailable(size int) bool {
|
||||
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", size)
|
||||
if _, err := os.Stat(hugepagesFile); err != nil {
|
||||
framework.Logf("Hugepages file %s not found: %v", hugepagesFile, err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// configureHugePages attempts to allocate hugepages of the specified size
|
||||
func ConfigureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error {
|
||||
// Compact memory to make bigger contiguous blocks of memory available
|
||||
// before allocating huge pages.
|
||||
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
|
||||
if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil {
|
||||
if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// e.g. hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile)
|
||||
|
||||
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix)
|
||||
if numaNodeID != nil {
|
||||
// e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
|
||||
hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix)
|
||||
}
|
||||
|
||||
// Reserve number of hugepages
|
||||
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
|
||||
command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile)
|
||||
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// verify that the number of hugepages was updated
|
||||
// e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
|
||||
command = fmt.Sprintf("cat %s", hugepagesFile)
|
||||
outData, err := exec.Command("/bin/sh", "-c", command).Output()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
framework.Logf("Hugepages total is set to %v", numHugePages)
|
||||
if numHugePages == hugepagesCount {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages)
|
||||
}
|
||||
|
||||
// TODO(KevinTMtz) - Deduplicate from test/e2e_node/util.go:restartKubelet
|
||||
func RestartKubelet(ctx context.Context, running bool) {
|
||||
kubeletServiceName := FindKubeletServiceName(running)
|
||||
// reset the kubelet service start-limit-hit
|
||||
stdout, err := exec.CommandContext(ctx, "sudo", "systemctl", "reset-failed", kubeletServiceName).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to reset kubelet start-limit-hit with systemctl: %v, %s", err, string(stdout))
|
||||
|
||||
stdout, err = exec.CommandContext(ctx, "sudo", "systemctl", "restart", kubeletServiceName).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to restart kubelet with systemctl: %v, %s", err, string(stdout))
|
||||
}
|
||||
|
||||
func FindKubeletServiceName(running bool) string {
|
||||
cmdLine := []string{
|
||||
"systemctl", "list-units", "*kubelet*",
|
||||
}
|
||||
if running {
|
||||
cmdLine = append(cmdLine, "--state=running")
|
||||
}
|
||||
stdout, err := exec.Command("sudo", cmdLine...).CombinedOutput()
|
||||
framework.ExpectNoError(err)
|
||||
regex := regexp.MustCompile("(kubelet-\\w+)")
|
||||
matches := regex.FindStringSubmatch(string(stdout))
|
||||
gomega.Expect(matches).ToNot(gomega.BeEmpty(), "Found more than one kubelet service running: %q", stdout)
|
||||
kubeletServiceName := matches[0]
|
||||
framework.Logf("Get running kubelet with systemctl: %v, %v", string(stdout), kubeletServiceName)
|
||||
return kubeletServiceName
|
||||
}
|
||||
|
||||
func WaitForHugepages(ctx context.Context, f *framework.Framework, hugepages map[string]int) {
|
||||
ginkgo.By("Waiting for hugepages resource to become available on the local node")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) error {
|
||||
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for hugepagesResource, count := range hugepages {
|
||||
capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)]
|
||||
if !ok {
|
||||
return fmt.Errorf("the node does not have the resource %s", hugepagesResource)
|
||||
}
|
||||
|
||||
size, succeed := capacity.AsInt64()
|
||||
if !succeed {
|
||||
return fmt.Errorf("failed to convert quantity to int64")
|
||||
}
|
||||
|
||||
expectedSize := count * resourceToSize[hugepagesResource] * 1024
|
||||
if size != int64(expectedSize) {
|
||||
return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, time.Minute, framework.Poll).Should(gomega.BeNil())
|
||||
}
|
||||
|
||||
func ReleaseHugepages(ctx context.Context, hugepages map[string]int) {
|
||||
ginkgo.By("Releasing hugepages")
|
||||
gomega.Eventually(ctx, func() error {
|
||||
for hugepagesResource := range hugepages {
|
||||
command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile)
|
||||
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
|
||||
}
|
||||
|
||||
// TODO(KevinTMtz) - Deduplicate from test/e2e/framework/skipper/skipper.go:Skipf
|
||||
func skipf(format string, args ...any) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
ginkgo.Skip(msg, 2)
|
||||
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user