Merge pull request #41964 from sttts/sttts-upgrade-test-sysctl

Automatic merge from submit-queue (batch tested with PRs 35408, 41915, 41992, 41964, 41925)

e2e/upgrade: add sysctls

Add sysctl upgrade tests.

How can these be effectively tested?

test/e2e/common/BUILD

@@ -40,7 +40,6 @@ go_library(
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/conditions:go_default_library",
         "//pkg/kubelet:go_default_library",
-        "//pkg/kubelet/events:go_default_library",
         "//pkg/kubelet/sysctl:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/utils:go_default_library",

test/e2e/common/sysctl.go

@@ -17,14 +17,9 @@ limitations under the License.
 package common
 
 import (
-	"fmt"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
-	"k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/pkg/kubelet/sysctl"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -57,27 +52,6 @@ var _ = framework.KubeDescribe("Sysctls", func() {
 		return &pod
 	}
 
-	waitForPodErrorEventOrStarted := func(pod *v1.Pod) (*v1.Event, error) {
-		var ev *v1.Event
-		err := wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) {
-			evnts, err := f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
-			if err != nil {
-				return false, fmt.Errorf("error in listing events: %s", err)
-			}
-			for _, e := range evnts.Items {
-				switch e.Reason {
-				case sysctl.UnsupportedReason, sysctl.ForbiddenReason:
-					ev = &e
-					return true, nil
-				case events.StartedContainer:
-					return true, nil
-				}
-			}
-			return false, nil
-		})
-		return ev, err
-	}
-
 	BeforeEach(func() {
 		podClient = f.PodClient()
 	})
@@ -99,7 +73,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
 		// watch for events instead of termination of pod because the kubelet deletes
 		// failed pods without running containers. This would create a race as the pod
 		// might have already been deleted here.
-		ev, err := waitForPodErrorEventOrStarted(pod)
+		ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
 		Expect(err).NotTo(HaveOccurred())
 		if ev != nil && ev.Reason == sysctl.UnsupportedReason {
 			framework.Skipf("No sysctl support in Docker <1.12")
@@ -140,7 +114,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
 		// watch for events instead of termination of pod because the kubelet deletes
 		// failed pods without running containers. This would create a race as the pod
 		// might have already been deleted here.
-		ev, err := waitForPodErrorEventOrStarted(pod)
+		ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
 		Expect(err).NotTo(HaveOccurred())
 		if ev != nil && ev.Reason == sysctl.UnsupportedReason {
 			framework.Skipf("No sysctl support in Docker <1.12")
@@ -222,7 +196,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
 		// watch for events instead of termination of pod because the kubelet deletes
 		// failed pods without running containers. This would create a race as the pod
 		// might have already been deleted here.
-		ev, err := waitForPodErrorEventOrStarted(pod)
+		ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
 		Expect(err).NotTo(HaveOccurred())
 		if ev != nil && ev.Reason == sysctl.UnsupportedReason {
 			framework.Skipf("No sysctl support in Docker <1.12")

test/e2e/framework/BUILD

@@ -60,9 +60,11 @@ go_library(
         "//pkg/controller/node:go_default_library",
         "//pkg/kubectl:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
+        "//pkg/kubelet/events:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
         "//pkg/kubelet/server/remotecommand:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
+        "//pkg/kubelet/sysctl:go_default_library",
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//pkg/metrics:go_default_library",

test/e2e/framework/pods.go

@@ -27,8 +27,11 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
+	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/kubernetes/pkg/kubelet/sysctl"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -180,7 +183,7 @@ func (c *PodClient) mungeSpec(pod *v1.Pod) {
 }
 
 // TODO(random-liu): Move pod wait function into this file
-// WaitForSuccess waits for pod to success.
+// WaitForSuccess waits for pod to succeed.
 func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
 	f := c.f
 	Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
@@ -197,6 +200,30 @@ func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
 	)).To(Succeed(), "wait for pod %q to success", name)
 }
 
+// WaitForSuccess waits for pod to succeed or an error event for that pod.
+func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
+	var ev *v1.Event
+	err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
+		evnts, err := c.f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
+		if err != nil {
+			return false, fmt.Errorf("error in listing events: %s", err)
+		}
+		for _, e := range evnts.Items {
+			switch e.Reason {
+			case events.KillingContainer, events.FailedToCreateContainer, sysctl.UnsupportedReason, sysctl.ForbiddenReason:
+				ev = &e
+				return true, nil
+			case events.StartedContainer:
+				return true, nil
+			default:
+				// ignore all other errors
+			}
+		}
+		return false, nil
+	})
+	return ev, err
+}
+
 // MatchContainerOutput gest output of a container and match expected regexp in the output.
 func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
 	f := c.f

test/e2e/upgrades/BUILD

@@ -16,6 +16,7 @@ go_library(
         "secrets.go",
         "services.go",
         "statefulset.go",
+        "sysctl.go",
         "upgrade.go",
     ],
     tags = ["automanaged"],
@@ -24,10 +25,12 @@ go_library(
         "//pkg/apis/apps/v1beta1:go_default_library",
         "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/controller/deployment/util:go_default_library",
+        "//pkg/kubelet/sysctl:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//vendor:github.com/onsi/ginkgo",
         "//vendor:github.com/onsi/gomega",
+        "//vendor:k8s.io/apimachinery/pkg/api/errors",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/util/uuid",
         "//vendor:k8s.io/apimachinery/pkg/util/wait",

test/e2e/upgrades/sysctl.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrades
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/kubelet/sysctl"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// SecretUpgradeTest tests that a pod with sysctls runs before and after an upgrade. During
+// a master upgrade, the exact pod is expected to stay running. A pod with unsafe sysctls is
+// expected to keep failing before and after the upgrade.
+type SysctlUpgradeTest struct {
+	validPod   *v1.Pod
+	invalidPod *v1.Pod
+}
+
+// Setup creates two pods: one with safe sysctls, one with unsafe sysctls. It checks that the former
+// launched and the later is rejected.
+func (t *SysctlUpgradeTest) Setup(f *framework.Framework) {
+	t.validPod = t.verifySafeSysctlWork(f)
+	t.invalidPod = t.verifyUnsafeSysctlsAreRejected(f)
+}
+
+// Test waits for the upgrade to complete, and then verifies that a
+// pod can still consume the ConfigMap.
+func (t *SysctlUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
+	<-done
+	switch upgrade {
+	case MasterUpgrade:
+		By("Checking the safe sysctl pod keeps running on master upgrade")
+		pod, err := f.ClientSet.Core().Pods(t.validPod.Namespace).Get(t.validPod.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pod.Status.Phase).To(Equal(v1.PodRunning))
+	}
+
+	By("Checking the old unsafe sysctl pod was not suddenly started during an upgrade")
+	pod, err := f.ClientSet.Core().Pods(t.invalidPod.Namespace).Get(t.invalidPod.Name, metav1.GetOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		Expect(err).NotTo(HaveOccurred())
+	}
+	if err == nil {
+		Expect(pod.Status.Phase).NotTo(Equal(v1.PodRunning))
+	}
+
+	t.verifySafeSysctlWork(f)
+	t.verifyUnsafeSysctlsAreRejected(f)
+}
+
+// Teardown cleans up any remaining resources.
+func (t *SysctlUpgradeTest) Teardown(f *framework.Framework) {
+	// rely on the namespace deletion to clean up everything
+}
+
+func (t *SysctlUpgradeTest) verifySafeSysctlWork(f *framework.Framework) *v1.Pod {
+	By("Creating a pod with safe sysctls")
+	safeSysctl := "net.ipv4.ip_local_port_range"
+	safeSysctlValue := "1024 1042"
+	validPod := sysctlTestPod("valid-sysctls", map[string]string{safeSysctl: safeSysctlValue})
+	validPod = f.PodClient().Create(t.validPod)
+
+	By("Making sure the valid pod launches")
+	ev, err := f.PodClient().WaitForErrorEventOrSuccess(t.validPod)
+	Expect(err).NotTo(HaveOccurred())
+	if ev != nil && ev.Reason == sysctl.UnsupportedReason {
+		framework.Skipf("No sysctl support in Docker <1.12")
+	}
+	f.TestContainerOutput("pod with safe sysctl launched", t.validPod, 0, []string{fmt.Sprintf("%s = %s", safeSysctl, safeSysctlValue)})
+
+	return validPod
+}
+
+func (t *SysctlUpgradeTest) verifyUnsafeSysctlsAreRejected(f *framework.Framework) *v1.Pod {
+	By("Creating a pod with unsafe sysctls")
+	invalidPod := sysctlTestPod("valid-sysctls-"+string(uuid.NewUUID()), map[string]string{
+		"fs.mount-max": "1000000",
+	})
+	invalidPod = f.PodClient().Create(invalidPod)
+
+	By("Making sure the invalid pod failed")
+	ev, err := f.PodClient().WaitForErrorEventOrSuccess(invalidPod)
+	Expect(err).NotTo(HaveOccurred())
+	if ev != nil && ev.Reason == sysctl.UnsupportedReason {
+		framework.Skipf("No sysctl support in Docker <1.12")
+	}
+	Expect(ev.Reason).To(Equal(sysctl.ForbiddenReason))
+
+	return invalidPod
+}
+
+func sysctlTestPod(name string, sysctls map[string]string) *v1.Pod {
+	sysctlList := []v1.Sysctl{}
+	keys := []string{}
+	for k, v := range sysctls {
+		sysctlList = append(sysctlList, v1.Sysctl{Name: k, Value: v})
+		keys = append(keys, k)
+	}
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Annotations: map[string]string{
+				v1.SysctlsPodAnnotationKey: v1.PodAnnotationsFromSysctls(sysctlList),
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "test-container",
+					Image:   "gcr.io/google_containers/busybox:1.24",
+					Command: append([]string{"/bin/sysctl"}, keys...),
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+}