From ea52f59c8c0237348edcecfee547d8a6eabd51b7 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 23 Feb 2017 10:46:50 +0100 Subject: [PATCH] e2e/upgrade: add sysctls --- test/e2e/common/BUILD | 1 - test/e2e/common/sysctl.go | 32 +-------- test/e2e/framework/BUILD | 2 + test/e2e/framework/pods.go | 29 +++++++- test/e2e/upgrades/BUILD | 3 + test/e2e/upgrades/sysctl.go | 140 ++++++++++++++++++++++++++++++++++++ 6 files changed, 176 insertions(+), 31 deletions(-) create mode 100644 test/e2e/upgrades/sysctl.go diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index 113b4c92d3b..ea25dae023f 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -39,7 +39,6 @@ go_library( "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/conditions:go_default_library", "//pkg/kubelet:go_default_library", - "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/sysctl:go_default_library", "//test/e2e/framework:go_default_library", "//test/utils:go_default_library", diff --git a/test/e2e/common/sysctl.go b/test/e2e/common/sysctl.go index db9f8382897..0a5c44538cf 100644 --- a/test/e2e/common/sysctl.go +++ b/test/e2e/common/sysctl.go @@ -17,14 +17,9 @@ limitations under the License. package common import ( - "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/sysctl" "k8s.io/kubernetes/test/e2e/framework" @@ -57,27 +52,6 @@ var _ = framework.KubeDescribe("Sysctls", func() { return &pod } - waitForPodErrorEventOrStarted := func(pod *v1.Pod) (*v1.Event, error) { - var ev *v1.Event - err := wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) { - evnts, err := f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod) - if err != nil { - return false, fmt.Errorf("error in listing events: %s", err) - } - for _, e := range evnts.Items { - switch e.Reason { - case sysctl.UnsupportedReason, sysctl.ForbiddenReason: - ev = &e - return true, nil - case events.StartedContainer: - return true, nil - } - } - return false, nil - }) - return ev, err - } - BeforeEach(func() { podClient = f.PodClient() }) @@ -99,7 +73,7 @@ var _ = framework.KubeDescribe("Sysctls", func() { // watch for events instead of termination of pod because the kubelet deletes // failed pods without running containers. This would create a race as the pod // might have already been deleted here. - ev, err := waitForPodErrorEventOrStarted(pod) + ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod) Expect(err).NotTo(HaveOccurred()) if ev != nil && ev.Reason == sysctl.UnsupportedReason { framework.Skipf("No sysctl support in Docker <1.12") @@ -140,7 +114,7 @@ var _ = framework.KubeDescribe("Sysctls", func() { // watch for events instead of termination of pod because the kubelet deletes // failed pods without running containers. This would create a race as the pod // might have already been deleted here. - ev, err := waitForPodErrorEventOrStarted(pod) + ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod) Expect(err).NotTo(HaveOccurred()) if ev != nil && ev.Reason == sysctl.UnsupportedReason { framework.Skipf("No sysctl support in Docker <1.12") @@ -222,7 +196,7 @@ var _ = framework.KubeDescribe("Sysctls", func() { // watch for events instead of termination of pod because the kubelet deletes // failed pods without running containers. This would create a race as the pod // might have already been deleted here. - ev, err := waitForPodErrorEventOrStarted(pod) + ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod) Expect(err).NotTo(HaveOccurred()) if ev != nil && ev.Reason == sysctl.UnsupportedReason { framework.Skipf("No sysctl support in Docker <1.12") diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 44a6610a181..2e2dc593e40 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -58,9 +58,11 @@ go_library( "//pkg/controller/deployment/util:go_default_library", "//pkg/kubectl:go_default_library", "//pkg/kubelet/api/v1alpha1/stats:go_default_library", + "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/sysctl:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/metrics:go_default_library", diff --git a/test/e2e/framework/pods.go b/test/e2e/framework/pods.go index 669f0549087..e944e467752 100644 --- a/test/e2e/framework/pods.go +++ b/test/e2e/framework/pods.go @@ -27,8 +27,11 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" + "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/pkg/kubelet/sysctl" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -169,7 +172,7 @@ func (c *PodClient) mungeSpec(pod *v1.Pod) { } // TODO(random-liu): Move pod wait function into this file -// WaitForSuccess waits for pod to success. +// WaitForSuccess waits for pod to succeed. func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) { f := c.f Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout, @@ -186,6 +189,30 @@ func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) { )).To(Succeed(), "wait for pod %q to success", name) } +// WaitForSuccess waits for pod to succeed or an error event for that pod. +func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) { + var ev *v1.Event + err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) { + evnts, err := c.f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod) + if err != nil { + return false, fmt.Errorf("error in listing events: %s", err) + } + for _, e := range evnts.Items { + switch e.Reason { + case events.KillingContainer, events.FailedToCreateContainer, sysctl.UnsupportedReason, sysctl.ForbiddenReason: + ev = &e + return true, nil + case events.StartedContainer: + return true, nil + default: + // ignore all other errors + } + } + return false, nil + }) + return ev, err +} + // MatchContainerOutput gest output of a container and match expected regexp in the output. func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error { f := c.f diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index e36647ab13b..87f2086b353 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -16,6 +16,7 @@ go_library( "secrets.go", "services.go", "statefulset.go", + "sysctl.go", "upgrade.go", ], tags = ["automanaged"], @@ -24,10 +25,12 @@ go_library( "//pkg/apis/apps/v1beta1:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/controller/deployment/util:go_default_library", + "//pkg/kubelet/sysctl:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//vendor:github.com/onsi/ginkgo", "//vendor:github.com/onsi/gomega", + "//vendor:k8s.io/apimachinery/pkg/api/errors", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/util/uuid", "//vendor:k8s.io/apimachinery/pkg/util/wait", diff --git a/test/e2e/upgrades/sysctl.go b/test/e2e/upgrades/sysctl.go new file mode 100644 index 00000000000..d01caceb852 --- /dev/null +++ b/test/e2e/upgrades/sysctl.go @@ -0,0 +1,140 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrades + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/kubelet/sysctl" + + "k8s.io/kubernetes/test/e2e/framework" +) + +// SecretUpgradeTest tests that a pod with sysctls runs before and after an upgrade. During +// a master upgrade, the exact pod is expected to stay running. A pod with unsafe sysctls is +// expected to keep failing before and after the upgrade. +type SysctlUpgradeTest struct { + validPod *v1.Pod + invalidPod *v1.Pod +} + +// Setup creates two pods: one with safe sysctls, one with unsafe sysctls. It checks that the former +// launched and the later is rejected. +func (t *SysctlUpgradeTest) Setup(f *framework.Framework) { + t.validPod = t.verifySafeSysctlWork(f) + t.invalidPod = t.verifyUnsafeSysctlsAreRejected(f) +} + +// Test waits for the upgrade to complete, and then verifies that a +// pod can still consume the ConfigMap. +func (t *SysctlUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) { + <-done + switch upgrade { + case MasterUpgrade: + By("Checking the safe sysctl pod keeps running on master upgrade") + pod, err := f.ClientSet.Core().Pods(t.validPod.Namespace).Get(t.validPod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Status.Phase).To(Equal(v1.PodRunning)) + } + + By("Checking the old unsafe sysctl pod was not suddenly started during an upgrade") + pod, err := f.ClientSet.Core().Pods(t.invalidPod.Namespace).Get(t.invalidPod.Name, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } + if err == nil { + Expect(pod.Status.Phase).NotTo(Equal(v1.PodRunning)) + } + + t.verifySafeSysctlWork(f) + t.verifyUnsafeSysctlsAreRejected(f) +} + +// Teardown cleans up any remaining resources. +func (t *SysctlUpgradeTest) Teardown(f *framework.Framework) { + // rely on the namespace deletion to clean up everything +} + +func (t *SysctlUpgradeTest) verifySafeSysctlWork(f *framework.Framework) *v1.Pod { + By("Creating a pod with safe sysctls") + safeSysctl := "net.ipv4.ip_local_port_range" + safeSysctlValue := "1024 1042" + validPod := sysctlTestPod("valid-sysctls", map[string]string{safeSysctl: safeSysctlValue}) + validPod = f.PodClient().Create(t.validPod) + + By("Making sure the valid pod launches") + ev, err := f.PodClient().WaitForErrorEventOrSuccess(t.validPod) + Expect(err).NotTo(HaveOccurred()) + if ev != nil && ev.Reason == sysctl.UnsupportedReason { + framework.Skipf("No sysctl support in Docker <1.12") + } + f.TestContainerOutput("pod with safe sysctl launched", t.validPod, 0, []string{fmt.Sprintf("%s = %s", safeSysctl, safeSysctlValue)}) + + return validPod +} + +func (t *SysctlUpgradeTest) verifyUnsafeSysctlsAreRejected(f *framework.Framework) *v1.Pod { + By("Creating a pod with unsafe sysctls") + invalidPod := sysctlTestPod("valid-sysctls-"+string(uuid.NewUUID()), map[string]string{ + "fs.mount-max": "1000000", + }) + invalidPod = f.PodClient().Create(invalidPod) + + By("Making sure the invalid pod failed") + ev, err := f.PodClient().WaitForErrorEventOrSuccess(invalidPod) + Expect(err).NotTo(HaveOccurred()) + if ev != nil && ev.Reason == sysctl.UnsupportedReason { + framework.Skipf("No sysctl support in Docker <1.12") + } + Expect(ev.Reason).To(Equal(sysctl.ForbiddenReason)) + + return invalidPod +} + +func sysctlTestPod(name string, sysctls map[string]string) *v1.Pod { + sysctlList := []v1.Sysctl{} + keys := []string{} + for k, v := range sysctls { + sysctlList = append(sysctlList, v1.Sysctl{Name: k, Value: v}) + keys = append(keys, k) + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + v1.SysctlsPodAnnotationKey: v1.PodAnnotationsFromSysctls(sysctlList), + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Image: "gcr.io/google_containers/busybox:1.24", + Command: append([]string{"/bin/sysctl"}, keys...), + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } +}