diff --git a/test/utils/ktesting/assert.go b/test/utils/ktesting/assert.go new file mode 100644 index 00000000000..cf767231823 --- /dev/null +++ b/test/utils/ktesting/assert.go @@ -0,0 +1,183 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" +) + +// FailureError is an error where the error string is meant to be passed to +// [TContext.Fatal] directly, i.e. adding some prefix like "unexpected error" is not +// necessary. It is also not necessary to dump the error struct. +type FailureError struct { + Msg string + FullStackTrace string +} + +func (f FailureError) Error() string { + return f.Msg +} + +func (f FailureError) Backtrace() string { + return f.FullStackTrace +} + +func (f FailureError) Is(target error) bool { + return target == ErrFailure +} + +// ErrFailure is an empty error that can be wrapped to indicate that an error +// is a FailureError. It can also be used to test for a FailureError:. +// +// return fmt.Errorf("some problem%w", ErrFailure) +// ... +// err := someOperation() +// if errors.Is(err, ErrFailure) { +// ... +// } +var ErrFailure error = FailureError{} + +func expect(tCtx TContext, actual interface{}, extra ...interface{}) gomega.Assertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Expect(actual, extra...) +} + +func expectNoError(tCtx TContext, err error, explain ...interface{}) { + tCtx.Helper() + + description := buildDescription(explain) + + var failure FailureError + if errors.As(err, &failure) { + if backtrace := failure.Backtrace(); backtrace != "" { + if description != "" { + tCtx.Log(description) + } + tCtx.Logf("Failed at:\n %s", strings.ReplaceAll(backtrace, "\n", "\n ")) + } + if description != "" { + tCtx.Fatalf("%s: %s", description, err.Error()) + } + tCtx.Fatal(err.Error()) + } + + if description == "" { + description = "Unexpected error" + } + tCtx.Logf("%s: %s\n%s", description, format.Object(err, 1)) + tCtx.Fatalf("%s: %v", description, err.Error()) +} + +func buildDescription(explain ...interface{}) string { + switch len(explain) { + case 0: + return "" + case 1: + if describe, ok := explain[0].(func() string); ok { + return describe() + } + } + return fmt.Sprintf(explain[0].(string), explain[1:]...) +} + +// Eventually wraps [gomega.Eventually] such that a failure will be reported via +// TContext.Fatal. +// +// In contrast to [gomega.Eventually], the parameter is strongly typed. It must +// accept a TContext as first argument and return one value, the one which is +// then checked with the matcher. +// +// In contrast to direct usage of [gomega.Eventually], make additional +// assertions inside the callback is okay as long as they use the TContext that +// is passed in. For example, errors can be checked with ExpectNoError: +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +// +// If there is no value, then an error can be returned: +// +// cb := func(func(tCtx ktesting.TContext) error { +// err := doSomething(...) +// return err +// } +// tCtx.Eventually(cb).Should(gomega.Succeed(), "foobar should succeed") +// +// The default Gomega poll interval and timeout are used. Setting a specific +// timeout may be useful: +// +// tCtx.Eventually(cb).Timeout(5 * time.Second).Should(gomega.Succeed(), "foobar should succeed") +// +// Canceling the context in the callback only affects code in the callback. The +// context passed to Eventually is not getting canceled. To abort polling +// immediately because the expected condition is known to not be reached +// anymore, use [gomega.StopTrying]: +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// if errors.Is(err, SomeFinalErr) { +// gomega.StopTrying("permanent failure).Wrap(err).Now() +// } +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +// +// To poll again after some specific timeout, use [gomega.TryAgainAfter]. This is +// particularly useful in [Consistently] to ignore some intermittent error. +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// var intermittentErr SomeIntermittentError +// if errors.As(err, &intermittentErr) { +// gomega.TryAgainAfter(intermittentErr.RetryPeriod).Wrap(err).Now() +// } +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +func Eventually[T any](tCtx TContext, cb func(TContext) T) gomega.AsyncAssertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Eventually(tCtx, func(ctx context.Context) (val T, err error) { + tCtx := WithContext(tCtx, ctx) + tCtx, finalize := WithError(tCtx, &err) + defer finalize() + tCtx = WithCancel(tCtx) + return cb(tCtx), nil + }) +} + +// Consistently wraps [gomega.Consistently] the same way as [Eventually] wraps +// [gomega.Eventually]. +func Consistently[T any](tCtx TContext, cb func(TContext) T) gomega.AsyncAssertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Consistently(tCtx, func(ctx context.Context) (val T, err error) { + tCtx := WithContext(tCtx, ctx) + tCtx, finalize := WithError(tCtx, &err) + defer finalize() + return cb(tCtx), nil + }) +} diff --git a/test/utils/ktesting/assert_test.go b/test/utils/ktesting/assert_test.go new file mode 100644 index 00000000000..5099eeb38d1 --- /dev/null +++ b/test/utils/ktesting/assert_test.go @@ -0,0 +1,194 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "fmt" + "regexp" + "testing" + "time" + + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" +) + +func TestAsync(t *testing.T) { + for name, tc := range map[string]struct { + cb func(TContext) + expectNoFail bool + expectError string + expectDuration time.Duration + }{ + "eventually-timeout": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) int { + // Canceling here is a nop. + tCtx.Cancel("testing") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +Expected + : 0 +to equal + : 1`, + }, + "eventually-final": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + gomega.StopTrying("final error").Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Told to stop trying after x.y s. +final error`, + }, + "eventually-error": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + tCtx.Fatal("some error") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +The function passed to Eventually returned the following error: + <*errors.joinError | 0xXXXX>: + some error + { + errs: [ + <*errors.errorString | 0xXXXX>{s: "some error"}, + ], + }`, + }, + "eventually-success": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + return 1.0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectNoFail: true, + expectError: ``, + }, + "eventually-retry": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + gomega.TryAgainAfter(time.Millisecond).Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +told to try again after 1ms`, + }, + "consistently-timeout": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + // Canceling here is a nop. + tCtx.Cancel("testing") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Failed after x.y s. +Expected + : 0 +to equal + : 1`, + }, + "consistently-final": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + gomega.StopTrying("final error").Now() + tCtx.FailNow() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Told to stop trying after x.y s. +final error`, + }, + "consistently-error": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + tCtx.Fatal("some error") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Failed after x.y s. +The function passed to Consistently returned the following error: + <*errors.joinError | 0xXXXX>: + some error + { + errs: [ + <*errors.errorString | 0xXXXX>{s: "some error"}, + ], + }`, + }, + "consistently-success": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + return 1.0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectNoFail: true, + expectError: ``, + }, + "consistently-retry": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + gomega.TryAgainAfter(time.Millisecond).Wrap(errors.New("intermittent error")).Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out while waiting on TryAgainAfter after x.y s. +told to try again after 1ms: intermittent error`, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + tCtx := Init(t) + var err error + tCtx, finalize := WithError(tCtx, &err) + start := time.Now() + func() { + defer finalize() + tc.cb(tCtx) + }() + duration := time.Since(start) + assert.InDelta(t, tc.expectDuration.Seconds(), duration.Seconds(), 0.1, fmt.Sprintf("callback invocation duration %s", duration)) + assert.Equal(t, !tc.expectNoFail, tCtx.Failed(), "Failed()") + if tc.expectError == "" { + assert.NoError(t, err) + } else if assert.NotNil(t, err) { + t.Logf("Result:\n%s", err.Error()) + errMsg := err.Error() + errMsg = regexp.MustCompile(`[[:digit:]]+\.[[:digit:]]+s`).ReplaceAllString(errMsg, "x.y s") + errMsg = regexp.MustCompile(`0x[[:xdigit:]]+`).ReplaceAllString(errMsg, "0xXXXX") + assert.Equal(t, tc.expectError, errMsg) + } + }) + } +} diff --git a/test/utils/ktesting/clientcontext.go b/test/utils/ktesting/clientcontext.go new file mode 100644 index 00000000000..d07cbccf14a --- /dev/null +++ b/test/utils/ktesting/clientcontext.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "fmt" + + "github.com/onsi/gomega" + apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/klog/v2" +) + +// WithRESTConfig initializes all client-go clients with new clients +// created for the config. The current test name gets included in the UserAgent. +func WithRESTConfig(tCtx TContext, cfg *rest.Config) TContext { + cfg = rest.CopyConfig(cfg) + cfg.UserAgent = fmt.Sprintf("%s -- %s", rest.DefaultKubernetesUserAgent(), tCtx.Name()) + + cCtx := clientContext{ + TContext: tCtx, + restConfig: cfg, + client: clientset.NewForConfigOrDie(cfg), + dynamic: dynamic.NewForConfigOrDie(cfg), + apiextensions: apiextensions.NewForConfigOrDie(cfg), + } + + cachedDiscovery := memory.NewMemCacheClient(cCtx.client.Discovery()) + cCtx.restMapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscovery) + + return &cCtx +} + +// WithClients uses an existing config and clients. +func WithClients(tCtx TContext, cfg *rest.Config, mapper *restmapper.DeferredDiscoveryRESTMapper, client clientset.Interface, dynamic dynamic.Interface, apiextensions apiextensions.Interface) TContext { + return clientContext{ + TContext: tCtx, + restConfig: cfg, + restMapper: mapper, + client: client, + dynamic: dynamic, + apiextensions: apiextensions, + } +} + +type clientContext struct { + TContext + + restConfig *rest.Config + restMapper *restmapper.DeferredDiscoveryRESTMapper + client clientset.Interface + dynamic dynamic.Interface + apiextensions apiextensions.Interface +} + +func (cCtx clientContext) CleanupCtx(cb func(TContext)) { + cCtx.Helper() + cleanupCtx(cCtx, cb) +} + +func (cCtx clientContext) Expect(actual interface{}, extra ...interface{}) gomega.Assertion { + cCtx.Helper() + return expect(cCtx, actual, extra...) +} + +func (cCtx clientContext) ExpectNoError(err error, explain ...interface{}) { + cCtx.Helper() + expectNoError(cCtx, err, explain...) +} + +func (cCtx clientContext) Logger() klog.Logger { + return klog.FromContext(cCtx) +} + +func (cCtx clientContext) RESTConfig() *rest.Config { + if cCtx.restConfig == nil { + return nil + } + return rest.CopyConfig(cCtx.restConfig) +} + +func (cCtx clientContext) RESTMapper() *restmapper.DeferredDiscoveryRESTMapper { + return cCtx.restMapper +} + +func (cCtx clientContext) Client() clientset.Interface { + return cCtx.client +} + +func (cCtx clientContext) Dynamic() dynamic.Interface { + return cCtx.dynamic +} + +func (cCtx clientContext) APIExtensions() apiextensions.Interface { + return cCtx.apiextensions +} diff --git a/test/utils/ktesting/contexthelper.go b/test/utils/ktesting/contexthelper.go new file mode 100644 index 00000000000..28d2b245934 --- /dev/null +++ b/test/utils/ktesting/contexthelper.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "fmt" + "time" +) + +// cleanupErr creates a cause when canceling a context because the test has completed. +// It is a context.Canceled error. +func cleanupErr(testName string) error { + return canceledError(fmt.Sprintf("test %s is cleaning up", testName)) +} + +type canceledError string + +func (c canceledError) Error() string { return string(c) } + +func (c canceledError) Is(target error) bool { + return target == context.Canceled +} + +// withTimeout corresponds to [context.WithTimeout]. In contrast to +// [context.WithTimeout], it automatically cancels during test cleanup, provides +// the given cause when the deadline is reached, and its cancel function +// requires a cause. +func withTimeout(ctx context.Context, tb TB, timeout time.Duration, timeoutCause string) (context.Context, func(cause string)) { + tb.Helper() + + now := time.Now() + + cancelCtx, cancel := context.WithCancelCause(ctx) + after := time.NewTimer(timeout) + stopCtx, stop := context.WithCancel(ctx) // Only used internally, doesn't need a cause. + tb.Cleanup(func() { + cancel(cleanupErr(tb.Name())) + stop() + }) + go func() { + select { + case <-stopCtx.Done(): + after.Stop() + // No need to set a cause here. The cause or error of + // the parent context will be used. + case <-after.C: + cancel(canceledError(timeoutCause)) + } + }() + + // Determine which deadline is sooner: ours or that of our parent. + deadline := now.Add(timeout) + if parentDeadline, ok := ctx.Deadline(); ok { + if deadline.After(parentDeadline) { + deadline = parentDeadline + } + } + + // We always have a deadline. + return deadlineContext{Context: cancelCtx, deadline: deadline}, func(cause string) { + var cancelCause error + if cause != "" { + cancelCause = canceledError(cause) + } + cancel(cancelCause) + } +} + +type deadlineContext struct { + context.Context + deadline time.Time +} + +func (d deadlineContext) Deadline() (time.Time, bool) { + return d.deadline, true +} diff --git a/test/utils/ktesting/contexthelper_test.go b/test/utils/ktesting/contexthelper_test.go new file mode 100644 index 00000000000..548f99a975f --- /dev/null +++ b/test/utils/ktesting/contexthelper_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCleanupErr(t *testing.T) { + actual := cleanupErr(t.Name()) + if !errors.Is(actual, context.Canceled) { + t.Errorf("cleanupErr %T should be a %T", actual, context.Canceled) + } +} + +func TestCause(t *testing.T) { + timeoutCause := canceledError("I timed out") + parentCause := errors.New("parent canceled") + + t.Parallel() + for name, tt := range map[string]struct { + parentCtx context.Context + timeout time.Duration + sleep time.Duration + cancelCause string + expectErr, expectCause error + expectDeadline time.Duration + }{ + "nothing": { + parentCtx: context.Background(), + timeout: 5 * time.Millisecond, + sleep: time.Millisecond, + }, + "timeout": { + parentCtx: context.Background(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: canceledError(timeoutCause), + }, + "parent-canceled": { + parentCtx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: context.Canceled, + }, + "parent-cause": { + parentCtx: func() context.Context { + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(parentCause) + return ctx + }(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: parentCause, + }, + "deadline-no-parent": { + parentCtx: context.Background(), + timeout: time.Minute, + expectDeadline: time.Minute, + }, + "deadline-parent": { + parentCtx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + return ctx + }(), + timeout: 2 * time.Minute, + expectDeadline: time.Minute, + }, + "deadline-child": { + parentCtx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + return ctx + }(), + timeout: time.Minute, + expectDeadline: time.Minute, + }, + } { + tt := tt + t.Run(name, func(t *testing.T) { + ctx, cancel := withTimeout(tt.parentCtx, t, tt.timeout, timeoutCause.Error()) + if tt.cancelCause != "" { + cancel(tt.cancelCause) + } + if tt.expectDeadline != 0 { + actualDeadline, ok := ctx.Deadline() + if assert.True(t, ok, "should have had a deadline") { + assert.InDelta(t, time.Until(actualDeadline), tt.expectDeadline, float64(time.Second), "remaining time till Deadline()") + } + } + time.Sleep(tt.sleep) + actualErr := ctx.Err() + actualCause := context.Cause(ctx) + assert.Equal(t, tt.expectErr, actualErr, "ctx.Err()") + assert.Equal(t, tt.expectCause, actualCause, "context.Cause()") + + }) + } +} diff --git a/test/utils/ktesting/doc.go b/test/utils/ktesting/doc.go new file mode 100644 index 00000000000..3207f311bc8 --- /dev/null +++ b/test/utils/ktesting/doc.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ktesting is a wrapper around k8s.io/klog/v2/ktesting. In contrast +// to the klog package, this one is opinionated and tailored towards testing +// Kubernetes. +// +// Importing it +// - adds the -v command line flag +// - enables better dumping of complex datatypes +// - sets the default verbosity to 5 (can be changed with [SetDefaultVerbosity]) +// +// It also adds additional APIs and types for unit and integration tests +// which are too experimental for klog and/or are unrelated to logging. +// The ktesting package itself takes care of managing a test context +// with deadlines, timeouts, cancellation, and some common attributes +// as first-class members of the API. Sub-packages have additional APIs +// for propagating values via the context, implemented via [WithValue]. +package ktesting diff --git a/test/utils/ktesting/errorcontext.go b/test/utils/ktesting/errorcontext.go new file mode 100644 index 00000000000..a963b199e85 --- /dev/null +++ b/test/utils/ktesting/errorcontext.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "fmt" + "strings" + "sync" + + "k8s.io/klog/v2" +) + +// WithError creates a context where test failures are collected and stored in +// the provided error instance when the caller is done. Use it like this: +// +// func doSomething(tCtx ktesting.TContext) (finalErr error) { +// tCtx, finalize := WithError(tCtx, &finalErr) +// defer finalize() +// ... +// tCtx.Fatal("some failure") +// +// Any error already stored in the variable will get overwritten by finalize if +// there were test failures, otherwise the variable is left unchanged. +// If there were multiple test errors, then the error will wrap all of +// them with errors.Join. +// +// Test failures are not propagated to the parent context. +func WithError(tCtx TContext, err *error) (TContext, func()) { + eCtx := &errorContext{ + TContext: tCtx, + } + + return eCtx, func() { + // Recover has to be called in the deferred function. When called inside + // a function called by a deferred function (like finalize below), it + // returns nil. + if e := recover(); e != nil { + if _, ok := e.(fatalWithError); !ok { + // Not our own panic, pass it on instead of setting the error. + panic(e) + } + } + + eCtx.finalize(err) + } +} + +type errorContext struct { + TContext + + mutex sync.Mutex + errors []error + failed bool +} + +func (eCtx *errorContext) finalize(err *error) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + if !eCtx.failed { + return + } + + errs := eCtx.errors + if len(errs) == 0 { + errs = []error{errFailedWithNoExplanation} + } + *err = errors.Join(errs...) +} + +func (eCtx *errorContext) Error(args ...any) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + // Gomega adds a leading newline in https://github.com/onsi/gomega/blob/f804ac6ada8d36164ecae0513295de8affce1245/internal/gomega.go#L37 + // Let's strip that at start and end because ktesting will make errors + // stand out more with the "ERROR" prefix, so there's no need for additional + // line breaks. + eCtx.errors = append(eCtx.errors, errors.New(strings.TrimSpace(fmt.Sprintln(args...)))) + eCtx.failed = true +} + +func (eCtx *errorContext) Errorf(format string, args ...any) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + eCtx.errors = append(eCtx.errors, errors.New(strings.TrimSpace(fmt.Sprintf(format, args...)))) + eCtx.failed = true +} + +func (eCtx *errorContext) Fail() { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + eCtx.failed = true +} + +func (eCtx *errorContext) FailNow() { + eCtx.Helper() + eCtx.Fail() + panic(failed) +} + +func (eCtx *errorContext) Failed() bool { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + return eCtx.failed +} + +func (eCtx *errorContext) Fatal(args ...any) { + eCtx.Error(args...) + eCtx.FailNow() +} + +func (eCtx *errorContext) Fatalf(format string, args ...any) { + eCtx.Errorf(format, args...) + eCtx.FailNow() +} + +func (eCtx *errorContext) CleanupCtx(cb func(TContext)) { + eCtx.Helper() + cleanupCtx(eCtx, cb) +} + +func (eCtx *errorContext) Logger() klog.Logger { + return klog.FromContext(eCtx) +} + +// fatalWithError is the internal type that should never get propagated up. The +// only case where that can happen is when the developer forgot to call +// finalize via defer. The string explains that, in case that developers get to +// see it. +type fatalWithError string + +const failed = fatalWithError("WithError TContext encountered a fatal error, but the finalize function was not called via defer as it should have been.") + +var errFailedWithNoExplanation = errors.New("WithError context was marked as failed without recording an error") diff --git a/test/utils/ktesting/errorcontext_test.go b/test/utils/ktesting/errorcontext_test.go new file mode 100644 index 00000000000..24492d9353d --- /dev/null +++ b/test/utils/ktesting/errorcontext_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithError(t *testing.T) { + t.Run("panic", func(t *testing.T) { + assert.Panics(t, func() { + tCtx := Init(t) + var err error + _, finalize := WithError(tCtx, &err) + defer finalize() + + panic("pass me through") + }) + }) + + normalErr := errors.New("normal error") + + for name, tc := range map[string]struct { + cb func(TContext) + expectNoFail bool + expectError string + }{ + "none": { + cb: func(tCtx TContext) {}, + expectNoFail: true, + expectError: normalErr.Error(), + }, + "Error": { + cb: func(tCtx TContext) { + tCtx.Error("some error") + }, + expectError: "some error", + }, + "Errorf": { + cb: func(tCtx TContext) { + tCtx.Errorf("some %s", "error") + }, + expectError: "some error", + }, + "Fatal": { + cb: func(tCtx TContext) { + tCtx.Fatal("some error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "Fatalf": { + cb: func(tCtx TContext) { + tCtx.Fatalf("some %s", "error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "Fail": { + cb: func(tCtx TContext) { + tCtx.Fatalf("some %s", "error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "FailNow": { + cb: func(tCtx TContext) { + tCtx.FailNow() + tCtx.Error("another error") + }, + expectError: errFailedWithNoExplanation.Error(), + }, + "many": { + cb: func(tCtx TContext) { + tCtx.Error("first error") + tCtx.Error("second error") + }, + expectError: `first error +second error`, + }, + } { + t.Run(name, func(t *testing.T) { + tCtx := Init(t) + err := normalErr + tCtx, finalize := WithError(tCtx, &err) + func() { + defer finalize() + tc.cb(tCtx) + }() + + assert.Equal(t, !tc.expectNoFail, tCtx.Failed(), "Failed()") + if tc.expectError == "" { + assert.NoError(t, err) + } else if assert.NotNil(t, err) { + assert.Equal(t, tc.expectError, err.Error()) + } + }) + } +} diff --git a/test/utils/ktesting/examples/gomega/doc.go b/test/utils/ktesting/examples/gomega/doc.go new file mode 100644 index 00000000000..76312bb4eb0 --- /dev/null +++ b/test/utils/ktesting/examples/gomega/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package gomega diff --git a/test/utils/ktesting/examples/gomega/example_test.go b/test/utils/ktesting/examples/gomega/example_test.go new file mode 100644 index 00000000000..48150766f86 --- /dev/null +++ b/test/utils/ktesting/examples/gomega/example_test.go @@ -0,0 +1,44 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gomega + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "context" + "testing" + "time" + + "github.com/onsi/gomega" + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestGomega(t *testing.T) { + tCtx := ktesting.Init(t) + + gomega.NewWithT(tCtx).Eventually(tCtx, func(ctx context.Context) int { + // TODO: tCtx = ktesting.WithContext(tCtx, ctx) + // Or some dedicated tCtx.Eventually? + + return 42 + }).WithPolling(time.Second).Should(gomega.Equal(1)) +} diff --git a/test/utils/ktesting/examples/logging/doc.go b/test/utils/ktesting/examples/logging/doc.go new file mode 100644 index 00000000000..67085e2d5f0 --- /dev/null +++ b/test/utils/ktesting/examples/logging/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package logging diff --git a/test/utils/ktesting/examples/logging/example_test.go b/test/utils/ktesting/examples/logging/example_test.go new file mode 100644 index 00000000000..6202838e56e --- /dev/null +++ b/test/utils/ktesting/examples/logging/example_test.go @@ -0,0 +1,74 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "testing" + + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestError(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Error("some", "thing") +} + +func TestErrorf(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Errorf("some %s", "thing") +} + +func TestFatal(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Fatal("some", "thing") + tCtx.Log("not reached") +} + +func TestFatalf(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Fatalf("some %s", "thing") + tCtx.Log("not reached") +} + +func TestInfo(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Log("hello via Log") + tCtx.Logger().Info("hello via Info") + tCtx.Error("some", "thing") +} + +func TestWithStep(t *testing.T) { + tCtx := ktesting.Init(t) + bake(ktesting.WithStep(tCtx, "bake cake")) +} + +func bake(tCtx ktesting.TContext) { + heatOven(ktesting.WithStep(tCtx, "set heat for baking")) +} + +func heatOven(tCtx ktesting.TContext) { + tCtx.Log("Log()") + tCtx.Logger().Info("Logger().Info()") + tCtx.Fatal("oven not found") +} diff --git a/test/utils/ktesting/examples/with_ktesting/doc.go b/test/utils/ktesting/examples/with_ktesting/doc.go new file mode 100644 index 00000000000..42b2a6b42fb --- /dev/null +++ b/test/utils/ktesting/examples/with_ktesting/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package withktesting diff --git a/test/utils/ktesting/examples/with_ktesting/example_test.go b/test/utils/ktesting/examples/with_ktesting/example_test.go new file mode 100644 index 00000000000..576f310c34c --- /dev/null +++ b/test/utils/ktesting/examples/with_ktesting/example_test.go @@ -0,0 +1,52 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package withktesting + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "context" + "testing" + "time" + + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestTimeout(t *testing.T) { + tCtx := ktesting.Init(t) + tmp := t.TempDir() + tCtx.Logf("Using %q as temporary directory.", tmp) + tCtx.Cleanup(func() { + t.Log("Cleaning up...") + }) + if deadline, ok := t.Deadline(); ok { + t.Logf("Will fail shortly before the test suite deadline at %s.", deadline) + } + select { + case <-time.After(1000 * time.Hour): + // This should not be reached. + tCtx.Log("Huh?! I shouldn't be that old.") + case <-tCtx.Done(): + // But this will before the test suite timeout. + tCtx.Errorf("need to stop: %v", context.Cause(tCtx)) + } +} diff --git a/test/utils/ktesting/examples/without_ktesting/doc.go b/test/utils/ktesting/examples/without_ktesting/doc.go new file mode 100644 index 00000000000..9b02e894aa3 --- /dev/null +++ b/test/utils/ktesting/examples/without_ktesting/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package withoutktesting diff --git a/test/utils/ktesting/examples/without_ktesting/example_test.go b/test/utils/ktesting/examples/without_ktesting/example_test.go new file mode 100644 index 00000000000..d0bc37eac3b --- /dev/null +++ b/test/utils/ktesting/examples/without_ktesting/example_test.go @@ -0,0 +1,40 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package withoutktesting + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "testing" + "time" +) + +func TestTimeout(t *testing.T) { + tmp := t.TempDir() + t.Logf("Using %q as temporary directory.", tmp) + t.Cleanup(func() { + t.Log("Cleaning up...") + }) + // This will not complete anytime soon... + t.Log("Please kill me.") + <-time.After(1000 * time.Hour) +} diff --git a/test/utils/ktesting/initoption/initoption.go b/test/utils/ktesting/initoption/initoption.go new file mode 100644 index 00000000000..cdac2be3950 --- /dev/null +++ b/test/utils/ktesting/initoption/initoption.go @@ -0,0 +1,30 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initoption + +import "k8s.io/kubernetes/test/utils/ktesting/internal" + +// InitOption is a functional option for Init and InitCtx. +type InitOption func(c *internal.InitConfig) + +// PerTestOutput controls whether a per-test logger gets +// set up by Init. Has no effect in InitCtx. +func PerTestOutput(enabled bool) InitOption { + return func(c *internal.InitConfig) { + c.PerTestOutput = enabled + } +} diff --git a/test/utils/ktesting/internal/config.go b/test/utils/ktesting/internal/config.go new file mode 100644 index 00000000000..22f7560aedf --- /dev/null +++ b/test/utils/ktesting/internal/config.go @@ -0,0 +1,21 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +type InitConfig struct { + PerTestOutput bool +} diff --git a/test/utils/ktesting/klogcontext.go b/test/utils/ktesting/klogcontext.go new file mode 100644 index 00000000000..08cd8e7e6be --- /dev/null +++ b/test/utils/ktesting/klogcontext.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "fmt" + "strings" + "time" +) + +var timeNow = time.Now // Can be stubbed out for testing. + +// withKlogHeader creates a TB where the same "I