mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #41964 from sttts/sttts-upgrade-test-sysctl
Automatic merge from submit-queue (batch tested with PRs 35408, 41915, 41992, 41964, 41925) e2e/upgrade: add sysctls Add sysctl upgrade tests. How can these be effectively tested?
This commit is contained in:
commit
f1fecc3074
@ -40,7 +40,6 @@ go_library(
|
||||
"//pkg/client/clientset_generated/internalclientset:go_default_library",
|
||||
"//pkg/client/conditions:go_default_library",
|
||||
"//pkg/kubelet:go_default_library",
|
||||
"//pkg/kubelet/events:go_default_library",
|
||||
"//pkg/kubelet/sysctl:go_default_library",
|
||||
"//test/e2e/framework:go_default_library",
|
||||
"//test/utils:go_default_library",
|
||||
|
@ -17,14 +17,9 @@ limitations under the License.
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
"k8s.io/kubernetes/pkg/api/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/events"
|
||||
"k8s.io/kubernetes/pkg/kubelet/sysctl"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
|
||||
@ -57,27 +52,6 @@ var _ = framework.KubeDescribe("Sysctls", func() {
|
||||
return &pod
|
||||
}
|
||||
|
||||
waitForPodErrorEventOrStarted := func(pod *v1.Pod) (*v1.Event, error) {
|
||||
var ev *v1.Event
|
||||
err := wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) {
|
||||
evnts, err := f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error in listing events: %s", err)
|
||||
}
|
||||
for _, e := range evnts.Items {
|
||||
switch e.Reason {
|
||||
case sysctl.UnsupportedReason, sysctl.ForbiddenReason:
|
||||
ev = &e
|
||||
return true, nil
|
||||
case events.StartedContainer:
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
return ev, err
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
podClient = f.PodClient()
|
||||
})
|
||||
@ -99,7 +73,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
|
||||
// watch for events instead of termination of pod because the kubelet deletes
|
||||
// failed pods without running containers. This would create a race as the pod
|
||||
// might have already been deleted here.
|
||||
ev, err := waitForPodErrorEventOrStarted(pod)
|
||||
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
|
||||
framework.Skipf("No sysctl support in Docker <1.12")
|
||||
@ -140,7 +114,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
|
||||
// watch for events instead of termination of pod because the kubelet deletes
|
||||
// failed pods without running containers. This would create a race as the pod
|
||||
// might have already been deleted here.
|
||||
ev, err := waitForPodErrorEventOrStarted(pod)
|
||||
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
|
||||
framework.Skipf("No sysctl support in Docker <1.12")
|
||||
@ -222,7 +196,7 @@ var _ = framework.KubeDescribe("Sysctls", func() {
|
||||
// watch for events instead of termination of pod because the kubelet deletes
|
||||
// failed pods without running containers. This would create a race as the pod
|
||||
// might have already been deleted here.
|
||||
ev, err := waitForPodErrorEventOrStarted(pod)
|
||||
ev, err := f.PodClient().WaitForErrorEventOrSuccess(pod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
|
||||
framework.Skipf("No sysctl support in Docker <1.12")
|
||||
|
@ -60,9 +60,11 @@ go_library(
|
||||
"//pkg/controller/node:go_default_library",
|
||||
"//pkg/kubectl:go_default_library",
|
||||
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",
|
||||
"//pkg/kubelet/events:go_default_library",
|
||||
"//pkg/kubelet/metrics:go_default_library",
|
||||
"//pkg/kubelet/server/remotecommand:go_default_library",
|
||||
"//pkg/kubelet/server/stats:go_default_library",
|
||||
"//pkg/kubelet/sysctl:go_default_library",
|
||||
"//pkg/kubelet/util/format:go_default_library",
|
||||
"//pkg/master/ports:go_default_library",
|
||||
"//pkg/metrics:go_default_library",
|
||||
|
@ -27,8 +27,11 @@ import (
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
"k8s.io/kubernetes/pkg/api/v1"
|
||||
v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/events"
|
||||
"k8s.io/kubernetes/pkg/kubelet/sysctl"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
@ -180,7 +183,7 @@ func (c *PodClient) mungeSpec(pod *v1.Pod) {
|
||||
}
|
||||
|
||||
// TODO(random-liu): Move pod wait function into this file
|
||||
// WaitForSuccess waits for pod to success.
|
||||
// WaitForSuccess waits for pod to succeed.
|
||||
func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
|
||||
f := c.f
|
||||
Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
|
||||
@ -197,6 +200,30 @@ func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
|
||||
)).To(Succeed(), "wait for pod %q to success", name)
|
||||
}
|
||||
|
||||
// WaitForSuccess waits for pod to succeed or an error event for that pod.
|
||||
func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
|
||||
var ev *v1.Event
|
||||
err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
|
||||
evnts, err := c.f.ClientSet.Core().Events(pod.Namespace).Search(api.Scheme, pod)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error in listing events: %s", err)
|
||||
}
|
||||
for _, e := range evnts.Items {
|
||||
switch e.Reason {
|
||||
case events.KillingContainer, events.FailedToCreateContainer, sysctl.UnsupportedReason, sysctl.ForbiddenReason:
|
||||
ev = &e
|
||||
return true, nil
|
||||
case events.StartedContainer:
|
||||
return true, nil
|
||||
default:
|
||||
// ignore all other errors
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
return ev, err
|
||||
}
|
||||
|
||||
// MatchContainerOutput gest output of a container and match expected regexp in the output.
|
||||
func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
|
||||
f := c.f
|
||||
|
@ -16,6 +16,7 @@ go_library(
|
||||
"secrets.go",
|
||||
"services.go",
|
||||
"statefulset.go",
|
||||
"sysctl.go",
|
||||
"upgrade.go",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
@ -24,10 +25,12 @@ go_library(
|
||||
"//pkg/apis/apps/v1beta1:go_default_library",
|
||||
"//pkg/apis/extensions/v1beta1:go_default_library",
|
||||
"//pkg/controller/deployment/util:go_default_library",
|
||||
"//pkg/kubelet/sysctl:go_default_library",
|
||||
"//test/e2e/common:go_default_library",
|
||||
"//test/e2e/framework:go_default_library",
|
||||
"//vendor:github.com/onsi/ginkgo",
|
||||
"//vendor:github.com/onsi/gomega",
|
||||
"//vendor:k8s.io/apimachinery/pkg/api/errors",
|
||||
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
|
||||
"//vendor:k8s.io/apimachinery/pkg/util/uuid",
|
||||
"//vendor:k8s.io/apimachinery/pkg/util/wait",
|
||||
|
140
test/e2e/upgrades/sysctl.go
Normal file
140
test/e2e/upgrades/sysctl.go
Normal file
@ -0,0 +1,140 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package upgrades
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/kubernetes/pkg/api/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/sysctl"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
)
|
||||
|
||||
// SecretUpgradeTest tests that a pod with sysctls runs before and after an upgrade. During
|
||||
// a master upgrade, the exact pod is expected to stay running. A pod with unsafe sysctls is
|
||||
// expected to keep failing before and after the upgrade.
|
||||
type SysctlUpgradeTest struct {
|
||||
validPod *v1.Pod
|
||||
invalidPod *v1.Pod
|
||||
}
|
||||
|
||||
// Setup creates two pods: one with safe sysctls, one with unsafe sysctls. It checks that the former
|
||||
// launched and the later is rejected.
|
||||
func (t *SysctlUpgradeTest) Setup(f *framework.Framework) {
|
||||
t.validPod = t.verifySafeSysctlWork(f)
|
||||
t.invalidPod = t.verifyUnsafeSysctlsAreRejected(f)
|
||||
}
|
||||
|
||||
// Test waits for the upgrade to complete, and then verifies that a
|
||||
// pod can still consume the ConfigMap.
|
||||
func (t *SysctlUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
|
||||
<-done
|
||||
switch upgrade {
|
||||
case MasterUpgrade:
|
||||
By("Checking the safe sysctl pod keeps running on master upgrade")
|
||||
pod, err := f.ClientSet.Core().Pods(t.validPod.Namespace).Get(t.validPod.Name, metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(pod.Status.Phase).To(Equal(v1.PodRunning))
|
||||
}
|
||||
|
||||
By("Checking the old unsafe sysctl pod was not suddenly started during an upgrade")
|
||||
pod, err := f.ClientSet.Core().Pods(t.invalidPod.Namespace).Get(t.invalidPod.Name, metav1.GetOptions{})
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
if err == nil {
|
||||
Expect(pod.Status.Phase).NotTo(Equal(v1.PodRunning))
|
||||
}
|
||||
|
||||
t.verifySafeSysctlWork(f)
|
||||
t.verifyUnsafeSysctlsAreRejected(f)
|
||||
}
|
||||
|
||||
// Teardown cleans up any remaining resources.
|
||||
func (t *SysctlUpgradeTest) Teardown(f *framework.Framework) {
|
||||
// rely on the namespace deletion to clean up everything
|
||||
}
|
||||
|
||||
func (t *SysctlUpgradeTest) verifySafeSysctlWork(f *framework.Framework) *v1.Pod {
|
||||
By("Creating a pod with safe sysctls")
|
||||
safeSysctl := "net.ipv4.ip_local_port_range"
|
||||
safeSysctlValue := "1024 1042"
|
||||
validPod := sysctlTestPod("valid-sysctls", map[string]string{safeSysctl: safeSysctlValue})
|
||||
validPod = f.PodClient().Create(t.validPod)
|
||||
|
||||
By("Making sure the valid pod launches")
|
||||
ev, err := f.PodClient().WaitForErrorEventOrSuccess(t.validPod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
|
||||
framework.Skipf("No sysctl support in Docker <1.12")
|
||||
}
|
||||
f.TestContainerOutput("pod with safe sysctl launched", t.validPod, 0, []string{fmt.Sprintf("%s = %s", safeSysctl, safeSysctlValue)})
|
||||
|
||||
return validPod
|
||||
}
|
||||
|
||||
func (t *SysctlUpgradeTest) verifyUnsafeSysctlsAreRejected(f *framework.Framework) *v1.Pod {
|
||||
By("Creating a pod with unsafe sysctls")
|
||||
invalidPod := sysctlTestPod("valid-sysctls-"+string(uuid.NewUUID()), map[string]string{
|
||||
"fs.mount-max": "1000000",
|
||||
})
|
||||
invalidPod = f.PodClient().Create(invalidPod)
|
||||
|
||||
By("Making sure the invalid pod failed")
|
||||
ev, err := f.PodClient().WaitForErrorEventOrSuccess(invalidPod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if ev != nil && ev.Reason == sysctl.UnsupportedReason {
|
||||
framework.Skipf("No sysctl support in Docker <1.12")
|
||||
}
|
||||
Expect(ev.Reason).To(Equal(sysctl.ForbiddenReason))
|
||||
|
||||
return invalidPod
|
||||
}
|
||||
|
||||
func sysctlTestPod(name string, sysctls map[string]string) *v1.Pod {
|
||||
sysctlList := []v1.Sysctl{}
|
||||
keys := []string{}
|
||||
for k, v := range sysctls {
|
||||
sysctlList = append(sysctlList, v1.Sysctl{Name: k, Value: v})
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Annotations: map[string]string{
|
||||
v1.SysctlsPodAnnotationKey: v1.PodAnnotationsFromSysctls(sysctlList),
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "test-container",
|
||||
Image: "gcr.io/google_containers/busybox:1.24",
|
||||
Command: append([]string{"/bin/sysctl"}, keys...),
|
||||
},
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
},
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user