From 1e6281e4a2566d53bda1cb43a42979d9c9189f11 Mon Sep 17 00:00:00 2001 From: Sergey Kanzhelev Date: Mon, 13 Mar 2023 20:54:50 +0000 Subject: [PATCH] first iteration to add standalone mode --- test/e2e/framework/kubelet/config.go | 11 +- test/e2e/framework/test_context.go | 2 + test/e2e_node/e2e_node_suite_test.go | 23 +++- test/e2e_node/services/kubelet.go | 23 +++- test/e2e_node/standalone_test.go | 192 +++++++++++++++++++++++++++ test/e2e_node/util.go | 2 +- 6 files changed, 236 insertions(+), 17 deletions(-) create mode 100644 test/e2e_node/standalone_test.go diff --git a/test/e2e/framework/kubelet/config.go b/test/e2e/framework/kubelet/config.go index edc9e2d48c5..76159691b8c 100644 --- a/test/e2e/framework/kubelet/config.go +++ b/test/e2e/framework/kubelet/config.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "k8s.io/kubernetes/pkg/cluster/ports" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" @@ -37,8 +38,8 @@ import ( ) // GetCurrentKubeletConfig fetches the current Kubelet Config for the given node -func GetCurrentKubeletConfig(ctx context.Context, nodeName, namespace string, useProxy bool) (*kubeletconfig.KubeletConfiguration, error) { - resp := pollConfigz(ctx, 5*time.Minute, 5*time.Second, nodeName, namespace, useProxy) +func GetCurrentKubeletConfig(ctx context.Context, nodeName, namespace string, useProxy bool, standaloneMode bool) (*kubeletconfig.KubeletConfiguration, error) { + resp := pollConfigz(ctx, 5*time.Minute, 5*time.Second, nodeName, namespace, useProxy, standaloneMode) if len(resp) == 0 { return nil, fmt.Errorf("failed to fetch /configz from %q", nodeName) } @@ -50,7 +51,7 @@ func GetCurrentKubeletConfig(ctx context.Context, nodeName, namespace string, us } // returns a status 200 response from the /configz endpoint or nil if fails -func pollConfigz(ctx context.Context, timeout time.Duration, pollInterval time.Duration, nodeName, namespace string, useProxy bool) []byte { +func pollConfigz(ctx context.Context, timeout time.Duration, pollInterval time.Duration, nodeName, namespace string, useProxy bool, standaloneMode bool) []byte { endpoint := "" if useProxy { // start local proxy, so we can send graceful deletion over query string, rather than body parameter @@ -75,8 +76,10 @@ func pollConfigz(ctx context.Context, timeout time.Duration, pollInterval time.D framework.ExpectNoError(err) framework.Logf("http requesting node kubelet /configz") endpoint = fmt.Sprintf("http://127.0.0.1:%d/api/v1/nodes/%s/proxy/configz", port, nodeName) - } else { + } else if !standaloneMode { endpoint = fmt.Sprintf("%s/api/v1/nodes/%s/proxy/configz", framework.TestContext.Host, framework.TestContext.NodeName) + } else { + endpoint = fmt.Sprintf("https://127.0.0.1:%d/configz", ports.KubeletPort) } tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 8b9ae042677..92dc4f68371 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -258,6 +258,8 @@ type NodeTestContextType struct { RestartKubelet bool // ExtraEnvs is a map of environment names to values. ExtraEnvs map[string]string + // StandaloneMode indicates whether the test is running kubelet in a standalone mode. + StandaloneMode bool } // CloudConfig holds the cloud configuration for e2e test suites. diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 081f4f00acc..17e95cd8214 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -104,6 +104,7 @@ func registerNodeFlags(flags *flag.FlagSet) { flags.BoolVar(&framework.TestContext.RequireDevices, "require-devices", false, "If true, require device plugins to be installed in the running environment.") flags.Var(cliflag.NewMapStringBool(&featureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features.") flags.Var(cliflag.NewMapStringBool(&serviceFeatureGates), "service-feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features for API service.") + flags.BoolVar(&framework.TestContext.StandaloneMode, "standalone-mode", false, "If true, starts kubelet in standalone mode.") } func init() { @@ -231,8 +232,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func(ctx context.Context) []byte { klog.Infof("Running tests without starting services.") } - klog.Infof("Wait for the node to be ready") - waitForNodeReady(ctx) + if !framework.TestContext.StandaloneMode { + klog.Infof("Wait for the node to be ready") + waitForNodeReady(ctx) + } // Reference common test to make the import valid. commontest.CurrentSuite = commontest.NodeE2E @@ -319,12 +322,18 @@ func updateTestContext(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to get apiserver client: %w", err) } - // Update test context with current node object. - node, err := getNode(client) - if err != nil { - return fmt.Errorf("failed to get node: %w", err) + + if !framework.TestContext.StandaloneMode { + // Update test context with current node object. + node, err := getNode(client) + if err != nil { + return fmt.Errorf("failed to get node: %w", err) + } + framework.TestContext.NodeName = node.Name // Set node name from API server, it is already set to the computer name by default. } - framework.TestContext.NodeName = node.Name // Set node name. + + framework.Logf("Node name: %s", framework.TestContext.NodeName) + // Update test context with current kubelet configuration. // This assumes all tests which dynamically change kubelet configuration // must: 1) run in serial; 2) restore kubelet configuration after test. diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 7f1b3c24ed0..7a917bc5778 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -151,10 +151,17 @@ func baseKubeConfiguration(cfgPath string) (*kubeletconfig.KubeletConfiguration, func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error) { klog.Info("Starting kubelet") - // Build kubeconfig - kubeconfigPath, err := createKubeconfigCWD() - if err != nil { - return nil, err + framework.Logf("Standalone mode: %v", framework.TestContext.StandaloneMode) + + var kubeconfigPath string + + if !framework.TestContext.StandaloneMode { + var err error + // Build kubeconfig + kubeconfigPath, err = createKubeconfigCWD() + if err != nil { + return nil, err + } } // KubeletConfiguration file path @@ -253,8 +260,14 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error kc.SystemCgroups = "/system" } + + if !framework.TestContext.StandaloneMode { + cmdArgs = append(cmdArgs, + "--kubeconfig", kubeconfigPath, + ) + } + cmdArgs = append(cmdArgs, - "--kubeconfig", kubeconfigPath, "--root-dir", KubeletRootDirectory, "--v", LogVerbosityLevel, ) diff --git a/test/e2e_node/standalone_test.go b/test/e2e_node/standalone_test.go new file mode 100644 index 00000000000..8a22c58c1c3 --- /dev/null +++ b/test/e2e_node/standalone_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/cluster/ports" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + admissionapi "k8s.io/pod-security-admission/api" + + "github.com/onsi/ginkgo/v2" +) + +var _ = SIGDescribe("[Feature:StandaloneMode] ", func() { + f := framework.NewDefaultFramework("static-pod") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + ginkgo.It("can create a static Pod ", func(ctx context.Context) { + + var ns, podPath, staticPodName string + + ns = f.Namespace.Name + staticPodName = "static-pod-" + string(uuid.NewUUID()) + + podPath = framework.TestContext.KubeletConfig.StaticPodPath + + err := createBasicStaticPod(podPath, staticPodName, ns, + imageutils.GetE2EImage(imageutils.Nginx), v1.RestartPolicyAlways) + framework.ExpectNoError(err) + + file := staticPodPath(podPath, staticPodName, ns) + defer os.Remove(file) + + pod := pullPods(ctx, 1*time.Minute, 5*time.Second, staticPodName) + + framework.ExpectEqual(pod.Status.Phase, v1.PodRunning) + }) +}) + +func createBasicStaticPod(dir, name, namespace, image string, restart v1.RestartPolicy) error { + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: "init-1", + Image: busyboxImage, + Command: ExecCommand("init-1", execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: "regular1", + Image: busyboxImage, + Command: ExecCommand("regular1", execCommand{ + Delay: 1000, + ExitCode: 0, + }), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("15Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("15Mi"), + }, + }, + }, + }, + }, + } + + podYaml, err := kubeadmutil.MarshalToYaml(podSpec, v1.SchemeGroupVersion) + if err != nil { + return err + } + + file := staticPodPath(dir, name, namespace) + + f, err := os.OpenFile(file, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(podYaml) + return err +} + +// returns a status 200 response from the /configz endpoint or nil if fails +func pullPods(ctx context.Context, timeout time.Duration, pollInterval time.Duration, name string) *v1.Pod { + endpoint := fmt.Sprintf("http://127.0.0.1:%d/pods", ports.KubeletReadOnlyPort) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + req, err := http.NewRequest("GET", endpoint, nil) + framework.ExpectNoError(err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", framework.TestContext.BearerToken)) + req.Header.Add("Accept", "application/json") + + var pod *v1.Pod + err = wait.PollImmediateWithContext(ctx, pollInterval, timeout, func(ctx context.Context) (bool, error) { + resp, err := client.Do(req) + if err != nil { + framework.Logf("Failed to get /pods, retrying. Error: %v", err) + return false, nil + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + framework.Logf("/pods response status not 200, retrying. Response was: %+v", resp) + return false, nil + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + framework.Logf("failed to read body from /pods response, retrying. Error: %v", err) + return false, nil + } + + pods, err := decodePods(respBody) + framework.ExpectNoError(err) + + found := false + for _, p := range pods.Items { + if strings.Contains(p.Name, name) { + found = true + pod = &p + } + } + + if !found { + framework.Logf("Pod %s not found in /pods response, retrying. Pods were: %v", name, string(respBody)) + return false, nil + } + + return true, nil + }) + framework.ExpectNoError(err, "Failed to get pod %s from /pods", name) + + return pod +} + +// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type). +func decodePods(respBody []byte) (*v1.PodList, error) { + // This hack because /pods reports the following structure: + // {"kind":"PodList","apiVersion":"v1","metadata":{},"items":[{"metadata":{"name":"kube-dns-autoscaler-758c4689b9-htpqj","generateName":"kube-dns-autoscaler-758c4689b9-", + + var pods v1.PodList + err := json.Unmarshal(respBody, &pods) + if err != nil { + return nil, err + } + + return &pods, nil +} diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 82068aead32..e1de2fa32fe 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -164,7 +164,7 @@ func getV1NodeDevices(ctx context.Context) (*kubeletpodresourcesv1.ListPodResour // Returns the current KubeletConfiguration func getCurrentKubeletConfig(ctx context.Context) (*kubeletconfig.KubeletConfiguration, error) { // namespace only relevant if useProxy==true, so we don't bother - return e2ekubelet.GetCurrentKubeletConfig(ctx, framework.TestContext.NodeName, "", false) + return e2ekubelet.GetCurrentKubeletConfig(ctx, framework.TestContext.NodeName, "", false, framework.TestContext.StandaloneMode) } // Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context.