Merge pull request #41964 from sttts/sttts-upgrade-test-sysctl

Automatic merge from submit-queue (batch tested with PRs 35408, 41915, 41992, 41964, 41925)

e2e/upgrade: add sysctls

Add sysctl upgrade tests.

How can these be effectively tested?
This commit is contained in:
Kubernetes Submit Queue 2017-02-26 18:07:59 -08:00 committed by GitHub
commit f1fecc3074
6 changed files with 176 additions and 31 deletions

View File

@ -40,7 +40,6 @@ go_library(
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/client/conditions:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/events:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/utils:go_default_library",

View File

@ -17,14 +17,9 @@ limitations under the License.
package common
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/pkg/kubelet/sysctl"
"k8s.io/kubernetes/test/e2e/framework"
@ -57,27 +52,6 @@ var _ = framework.KubeDescribe("Sysctls", func() {
return &pod
}
waitForPodErrorEventOrStarted := func(pod *v1.Pod) (*v1.Event, error) {
var ev *v1.Event
err := wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) {
evnts, err := f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
if err != nil {
return false, fmt.Errorf("error in listing events: %s", err)
}
for _, e := range evnts.Items {
switch e.Reason {
case sysctl.UnsupportedReason, sysctl.ForbiddenReason:
ev = &e
return true, nil
case events.StartedContainer:
return true, nil
}
}
return false, nil
})
return ev, err
}
BeforeEach(func() {
podClient = f.PodClient()
})
@ -99,7 +73,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
// watch for events instead of termination of pod because the kubelet deletes
// failed pods without running containers. This would create a race as the pod
// might have already been deleted here.
ev, err := waitForPodErrorEventOrStarted(pod)
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
Expect(err).NotTo(HaveOccurred())
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
framework.Skipf("No sysctl support in Docker <1.12")
@ -140,7 +114,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
// watch for events instead of termination of pod because the kubelet deletes
// failed pods without running containers. This would create a race as the pod
// might have already been deleted here.
ev, err := waitForPodErrorEventOrStarted(pod)
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
Expect(err).NotTo(HaveOccurred())
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
framework.Skipf("No sysctl support in Docker <1.12")
@ -222,7 +196,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
// watch for events instead of termination of pod because the kubelet deletes
// failed pods without running containers. This would create a race as the pod
// might have already been deleted here.
ev, err := waitForPodErrorEventOrStarted(pod)
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
Expect(err).NotTo(HaveOccurred())
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
framework.Skipf("No sysctl support in Docker <1.12")

View File

@ -60,9 +60,11 @@ go_library(
"//pkg/controller/node:go_default_library",
"//pkg/kubectl:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",
"//pkg/kubelet/events:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/server/remotecommand:go_default_library",
"//pkg/kubelet/server/stats:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//pkg/kubelet/util/format:go_default_library",
"//pkg/master/ports:go_default_library",
"//pkg/metrics:go_default_library",

View File

@ -27,8 +27,11 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/pkg/kubelet/sysctl"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -180,7 +183,7 @@ func (c *PodClient) mungeSpec(pod *v1.Pod) {
}
// TODO(random-liu): Move pod wait function into this file
// WaitForSuccess waits for pod to success.
// WaitForSuccess waits for pod to succeed.
func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
f := c.f
Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
@ -197,6 +200,30 @@ func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
)).To(Succeed(), "wait for pod %q to success", name)
}
// WaitForSuccess waits for pod to succeed or an error event for that pod.
func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
var ev *v1.Event
err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
evnts, err := c.f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
if err != nil {
return false, fmt.Errorf("error in listing events: %s", err)
}
for _, e := range evnts.Items {
switch e.Reason {
case events.KillingContainer, events.FailedToCreateContainer, sysctl.UnsupportedReason, sysctl.ForbiddenReason:
ev = &e
return true, nil
case events.StartedContainer:
return true, nil
default:
// ignore all other errors
}
}
return false, nil
})
return ev, err
}
// MatchContainerOutput gest output of a container and match expected regexp in the output.
func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
f := c.f

View File

@ -16,6 +16,7 @@ go_library(
"secrets.go",
"services.go",
"statefulset.go",
"sysctl.go",
"upgrade.go",
],
tags = ["automanaged"],
@ -24,10 +25,12 @@ go_library(
"//pkg/apis/apps/v1beta1:go_default_library",
"//pkg/apis/extensions/v1beta1:go_default_library",
"//pkg/controller/deployment/util:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//vendor:github.com/onsi/ginkgo",
"//vendor:github.com/onsi/gomega",
"//vendor:k8s.io/apimachinery/pkg/api/errors",
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
"//vendor:k8s.io/apimachinery/pkg/util/uuid",
"//vendor:k8s.io/apimachinery/pkg/util/wait",

140
test/e2e/upgrades/sysctl.go Normal file
View File

@ -0,0 +1,140 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrades
import (
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/kubelet/sysctl"
"k8s.io/kubernetes/test/e2e/framework"
)
// SecretUpgradeTest tests that a pod with sysctls runs before and after an upgrade. During
// a master upgrade, the exact pod is expected to stay running. A pod with unsafe sysctls is
// expected to keep failing before and after the upgrade.
type SysctlUpgradeTest struct {
validPod *v1.Pod
invalidPod *v1.Pod
}
// Setup creates two pods: one with safe sysctls, one with unsafe sysctls. It checks that the former
// launched and the later is rejected.
func (t *SysctlUpgradeTest) Setup(f *framework.Framework) {
t.validPod = t.verifySafeSysctlWork(f)
t.invalidPod = t.verifyUnsafeSysctlsAreRejected(f)
}
// Test waits for the upgrade to complete, and then verifies that a
// pod can still consume the ConfigMap.
func (t *SysctlUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
<-done
switch upgrade {
case MasterUpgrade:
By("Checking the safe sysctl pod keeps running on master upgrade")
pod, err := f.ClientSet.Core().Pods(t.validPod.Namespace).Get(t.validPod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(pod.Status.Phase).To(Equal(v1.PodRunning))
}
By("Checking the old unsafe sysctl pod was not suddenly started during an upgrade")
pod, err := f.ClientSet.Core().Pods(t.invalidPod.Namespace).Get(t.invalidPod.Name, metav1.GetOptions{})
if err != nil && !errors.IsNotFound(err) {
Expect(err).NotTo(HaveOccurred())
}
if err == nil {
Expect(pod.Status.Phase).NotTo(Equal(v1.PodRunning))
}
t.verifySafeSysctlWork(f)
t.verifyUnsafeSysctlsAreRejected(f)
}
// Teardown cleans up any remaining resources.
func (t *SysctlUpgradeTest) Teardown(f *framework.Framework) {
// rely on the namespace deletion to clean up everything
}
func (t *SysctlUpgradeTest) verifySafeSysctlWork(f *framework.Framework) *v1.Pod {
By("Creating a pod with safe sysctls")
safeSysctl := "net.ipv4.ip_local_port_range"
safeSysctlValue := "1024 1042"
validPod := sysctlTestPod("valid-sysctls", map[string]string{safeSysctl: safeSysctlValue})
validPod = f.PodClient().Create(t.validPod)
By("Making sure the valid pod launches")
ev, err := f.PodClient().WaitForErrorEventOrSuccess(t.validPod)
Expect(err).NotTo(HaveOccurred())
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
framework.Skipf("No sysctl support in Docker <1.12")
}
f.TestContainerOutput("pod with safe sysctl launched", t.validPod, 0, []string{fmt.Sprintf("%s = %s", safeSysctl, safeSysctlValue)})
return validPod
}
func (t *SysctlUpgradeTest) verifyUnsafeSysctlsAreRejected(f *framework.Framework) *v1.Pod {
By("Creating a pod with unsafe sysctls")
invalidPod := sysctlTestPod("valid-sysctls-"+string(uuid.NewUUID()), map[string]string{
"fs.mount-max": "1000000",
})
invalidPod = f.PodClient().Create(invalidPod)
By("Making sure the invalid pod failed")
ev, err := f.PodClient().WaitForErrorEventOrSuccess(invalidPod)
Expect(err).NotTo(HaveOccurred())
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
framework.Skipf("No sysctl support in Docker <1.12")
}
Expect(ev.Reason).To(Equal(sysctl.ForbiddenReason))
return invalidPod
}
func sysctlTestPod(name string, sysctls map[string]string) *v1.Pod {
sysctlList := []v1.Sysctl{}
keys := []string{}
for k, v := range sysctls {
sysctlList = append(sysctlList, v1.Sysctl{Name: k, Value: v})
keys = append(keys, k)
}
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Annotations: map[string]string{
v1.SysctlsPodAnnotationKey: v1.PodAnnotationsFromSysctls(sysctlList),
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "test-container",
Image: "gcr.io/google_containers/busybox:1.24",
Command: append([]string{"/bin/sysctl"}, keys...),
},
},
RestartPolicy: v1.RestartPolicyNever,
},
}
}