mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-06 02:34:03 +00:00
The idea of how this test should work
Signed-off-by: Laura Lorenz <lauralorenz@google.com>
This commit is contained in:
parent
6337a28a68
commit
6ef05dbd01
@ -97,6 +97,7 @@ var _ = SIGDescribe(feature.CriProxy, framework.WithSerial(), func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
ginkgo.It("Image pull retry backs off on error.", func(ctx context.Context) {
|
ginkgo.It("Image pull retry backs off on error.", func(ctx context.Context) {
|
||||||
|
// inject PullImage failed to trigger backoff
|
||||||
expectedErr := fmt.Errorf("PullImage failed")
|
expectedErr := fmt.Errorf("PullImage failed")
|
||||||
err := addCRIProxyInjector(func(apiName string) error {
|
err := addCRIProxyInjector(func(apiName string) error {
|
||||||
if apiName == criproxy.PullImage {
|
if apiName == criproxy.PullImage {
|
||||||
@ -120,15 +121,22 @@ var _ = SIGDescribe(feature.CriProxy, framework.WithSerial(), func() {
|
|||||||
isExpectedErrMsg := strings.Contains(eventMsg, expectedErr.Error())
|
isExpectedErrMsg := strings.Contains(eventMsg, expectedErr.Error())
|
||||||
gomega.Expect(isExpectedErrMsg).To(gomega.BeTrueBecause("we injected an exception into the PullImage interface of the cri proxy"))
|
gomega.Expect(isExpectedErrMsg).To(gomega.BeTrueBecause("we injected an exception into the PullImage interface of the cri proxy"))
|
||||||
|
|
||||||
// remove error so after backoff we will succeed
|
// Wait for ~40s worth of backoffs to occur so we can confirm the backoff growth.
|
||||||
|
podErr = e2epod.WaitForPodContainerStarted(ctx, f.ClientSet, f.Namespace.Name, pod.Name, 0, 40*time.Second)
|
||||||
|
gomega.Expect(podErr).To(gomega.HaveOccurred(), "Expected container not to start from repeadedly backing off image pulls")
|
||||||
|
|
||||||
|
// Remove error so after next backoff we will succeed.
|
||||||
_ = resetCRIProxyInjector()
|
_ = resetCRIProxyInjector()
|
||||||
|
|
||||||
podErr = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
podErr = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
||||||
framework.ExpectNoError(podErr)
|
framework.ExpectNoError(podErr)
|
||||||
|
|
||||||
|
// Parse observed backoffs (TODO: don't use Events, but logs) and compare to expectations
|
||||||
durations, err := getImageBackOffDurations(ctx, f, pod.Name)
|
durations, err := getImageBackOffDurations(ctx, f, pod.Name)
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
|
gomega.Expect(durations).Error().ShouldNot(gomega.BeNil(), "Should have observed backoffs in Pod event log")
|
||||||
gomega.Expect(durations[0]).Should(gomega.BeNumerically("~", time.Duration(10*time.Second), time.Duration(2*time.Second)))
|
gomega.Expect(durations[0]).Should(gomega.BeNumerically("~", time.Duration(10*time.Second), time.Duration(2*time.Second)))
|
||||||
|
// TODO: and check the next set of durations are 2x, etc
|
||||||
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@ -191,25 +199,26 @@ func getImageBackOffDurations(ctx context.Context, f *framework.Framework, podNa
|
|||||||
var backoffs []time.Duration
|
var backoffs []time.Duration
|
||||||
|
|
||||||
type BackOffRecord struct {
|
type BackOffRecord struct {
|
||||||
podName string
|
|
||||||
initialEventTime time.Time
|
initialEventTime time.Time
|
||||||
backoffEventTimes []time.Time
|
backoffEventTimes []time.Time
|
||||||
duration time.Duration
|
duration time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
records := make(map[int]*BackOffRecord)
|
records := make(map[int]*BackOffRecord)
|
||||||
|
records[0] = &BackOffRecord{}
|
||||||
var backoffCount int
|
var backoffCount int
|
||||||
var pullTime time.Time
|
var pullTime time.Time
|
||||||
var r *BackOffRecord
|
var r *BackOffRecord
|
||||||
|
// I'm doing this here for events but really it needs to be off kubelet logs or some kind of synchronous watch
|
||||||
|
// Because the events normalize to the latest
|
||||||
|
// But this is the idea
|
||||||
for _, event := range events.Items {
|
for _, event := range events.Items {
|
||||||
if event.InvolvedObject.Name == podName {
|
if event.InvolvedObject.Name == podName {
|
||||||
|
|
||||||
switch event.Reason {
|
switch event.Reason {
|
||||||
case kubeletevents.PullingImage:
|
case kubeletevents.PullingImage:
|
||||||
if !pullTime.IsZero() {
|
if !pullTime.IsZero() {
|
||||||
if event.FirstTimestamp.Time.After(pullTime) {
|
if event.FirstTimestamp.Time.After(pullTime) {
|
||||||
r = records[backoffCount]
|
r = records[backoffCount]
|
||||||
r.podName = podName
|
|
||||||
r.duration = r.initialEventTime.Sub(r.backoffEventTimes[len(r.backoffEventTimes)-1])
|
r.duration = r.initialEventTime.Sub(r.backoffEventTimes[len(r.backoffEventTimes)-1])
|
||||||
backoffs = append(backoffs, r.duration)
|
backoffs = append(backoffs, r.duration)
|
||||||
backoffCount++
|
backoffCount++
|
||||||
|
Loading…
Reference in New Issue
Block a user