From 63aa2615834393acd277abccb3f79422cb0c4452 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Sun, 3 Dec 2023 18:22:00 +0100 Subject: [PATCH] ktesting: add TContext The new TContext interface combines a normal context and the testing interface, then adds some helper methods. The context gets canceled when the test is done, but that can also be requested earlier via Cancel. The intended usage is to pass a single `tCtx ktesting.TContext` parameter around in all helper functions that get called by a unit or integration test. Logging is also more useful: Log[f] and Fatal[f] output is prefixed with "[FATAL] ERROR: " to make it stand out more from regular log output. If this approach turns out to be useful, it could be extended further (for example, with a per-test timeout) and might get moved to a staging repository to enable usage of it in other staging repositories. To allow other implementations besides testing.T and testing.B, a custom ktesting.TB interface gets defined with the methods expected from the actual implementation. One such implementation can be ginkgo.GinkgoT(). --- test/utils/ktesting/assert.go | 183 +++++++ test/utils/ktesting/assert_test.go | 194 ++++++++ test/utils/ktesting/clientcontext.go | 114 +++++ test/utils/ktesting/contexthelper.go | 91 ++++ test/utils/ktesting/contexthelper_test.go | 126 +++++ test/utils/ktesting/doc.go | 32 ++ test/utils/ktesting/errorcontext.go | 153 ++++++ test/utils/ktesting/errorcontext_test.go | 116 +++++ test/utils/ktesting/examples/gomega/doc.go | 20 + .../ktesting/examples/gomega/example_test.go | 44 ++ test/utils/ktesting/examples/logging/doc.go | 20 + .../ktesting/examples/logging/example_test.go | 74 +++ .../ktesting/examples/with_ktesting/doc.go | 20 + .../examples/with_ktesting/example_test.go | 52 ++ .../ktesting/examples/without_ktesting/doc.go | 20 + .../examples/without_ktesting/example_test.go | 40 ++ test/utils/ktesting/initoption/initoption.go | 30 ++ test/utils/ktesting/internal/config.go | 21 + test/utils/ktesting/klogcontext.go | 76 +++ test/utils/ktesting/ktesting.go | 47 +- test/utils/ktesting/signals.go | 139 ++++++ test/utils/ktesting/signals_non_win.go | 31 ++ test/utils/ktesting/signals_win.go | 28 ++ test/utils/ktesting/stepcontext.go | 101 ++++ test/utils/ktesting/tcontext.go | 466 ++++++++++++++++++ test/utils/ktesting/tcontext_test.go | 83 ++++ test/utils/ktesting/withcontext.go | 123 +++++ 27 files changed, 2411 insertions(+), 33 deletions(-) create mode 100644 test/utils/ktesting/assert.go create mode 100644 test/utils/ktesting/assert_test.go create mode 100644 test/utils/ktesting/clientcontext.go create mode 100644 test/utils/ktesting/contexthelper.go create mode 100644 test/utils/ktesting/contexthelper_test.go create mode 100644 test/utils/ktesting/doc.go create mode 100644 test/utils/ktesting/errorcontext.go create mode 100644 test/utils/ktesting/errorcontext_test.go create mode 100644 test/utils/ktesting/examples/gomega/doc.go create mode 100644 test/utils/ktesting/examples/gomega/example_test.go create mode 100644 test/utils/ktesting/examples/logging/doc.go create mode 100644 test/utils/ktesting/examples/logging/example_test.go create mode 100644 test/utils/ktesting/examples/with_ktesting/doc.go create mode 100644 test/utils/ktesting/examples/with_ktesting/example_test.go create mode 100644 test/utils/ktesting/examples/without_ktesting/doc.go create mode 100644 test/utils/ktesting/examples/without_ktesting/example_test.go create mode 100644 test/utils/ktesting/initoption/initoption.go create mode 100644 test/utils/ktesting/internal/config.go create mode 100644 test/utils/ktesting/klogcontext.go create mode 100644 test/utils/ktesting/signals.go create mode 100644 test/utils/ktesting/signals_non_win.go create mode 100644 test/utils/ktesting/signals_win.go create mode 100644 test/utils/ktesting/stepcontext.go create mode 100644 test/utils/ktesting/tcontext.go create mode 100644 test/utils/ktesting/tcontext_test.go create mode 100644 test/utils/ktesting/withcontext.go diff --git a/test/utils/ktesting/assert.go b/test/utils/ktesting/assert.go new file mode 100644 index 00000000000..cf767231823 --- /dev/null +++ b/test/utils/ktesting/assert.go @@ -0,0 +1,183 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" +) + +// FailureError is an error where the error string is meant to be passed to +// [TContext.Fatal] directly, i.e. adding some prefix like "unexpected error" is not +// necessary. It is also not necessary to dump the error struct. +type FailureError struct { + Msg string + FullStackTrace string +} + +func (f FailureError) Error() string { + return f.Msg +} + +func (f FailureError) Backtrace() string { + return f.FullStackTrace +} + +func (f FailureError) Is(target error) bool { + return target == ErrFailure +} + +// ErrFailure is an empty error that can be wrapped to indicate that an error +// is a FailureError. It can also be used to test for a FailureError:. +// +// return fmt.Errorf("some problem%w", ErrFailure) +// ... +// err := someOperation() +// if errors.Is(err, ErrFailure) { +// ... +// } +var ErrFailure error = FailureError{} + +func expect(tCtx TContext, actual interface{}, extra ...interface{}) gomega.Assertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Expect(actual, extra...) +} + +func expectNoError(tCtx TContext, err error, explain ...interface{}) { + tCtx.Helper() + + description := buildDescription(explain) + + var failure FailureError + if errors.As(err, &failure) { + if backtrace := failure.Backtrace(); backtrace != "" { + if description != "" { + tCtx.Log(description) + } + tCtx.Logf("Failed at:\n %s", strings.ReplaceAll(backtrace, "\n", "\n ")) + } + if description != "" { + tCtx.Fatalf("%s: %s", description, err.Error()) + } + tCtx.Fatal(err.Error()) + } + + if description == "" { + description = "Unexpected error" + } + tCtx.Logf("%s: %s\n%s", description, format.Object(err, 1)) + tCtx.Fatalf("%s: %v", description, err.Error()) +} + +func buildDescription(explain ...interface{}) string { + switch len(explain) { + case 0: + return "" + case 1: + if describe, ok := explain[0].(func() string); ok { + return describe() + } + } + return fmt.Sprintf(explain[0].(string), explain[1:]...) +} + +// Eventually wraps [gomega.Eventually] such that a failure will be reported via +// TContext.Fatal. +// +// In contrast to [gomega.Eventually], the parameter is strongly typed. It must +// accept a TContext as first argument and return one value, the one which is +// then checked with the matcher. +// +// In contrast to direct usage of [gomega.Eventually], make additional +// assertions inside the callback is okay as long as they use the TContext that +// is passed in. For example, errors can be checked with ExpectNoError: +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +// +// If there is no value, then an error can be returned: +// +// cb := func(func(tCtx ktesting.TContext) error { +// err := doSomething(...) +// return err +// } +// tCtx.Eventually(cb).Should(gomega.Succeed(), "foobar should succeed") +// +// The default Gomega poll interval and timeout are used. Setting a specific +// timeout may be useful: +// +// tCtx.Eventually(cb).Timeout(5 * time.Second).Should(gomega.Succeed(), "foobar should succeed") +// +// Canceling the context in the callback only affects code in the callback. The +// context passed to Eventually is not getting canceled. To abort polling +// immediately because the expected condition is known to not be reached +// anymore, use [gomega.StopTrying]: +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// if errors.Is(err, SomeFinalErr) { +// gomega.StopTrying("permanent failure).Wrap(err).Now() +// } +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +// +// To poll again after some specific timeout, use [gomega.TryAgainAfter]. This is +// particularly useful in [Consistently] to ignore some intermittent error. +// +// cb := func(func(tCtx ktesting.TContext) int { +// value, err := doSomething(...) +// var intermittentErr SomeIntermittentError +// if errors.As(err, &intermittentErr) { +// gomega.TryAgainAfter(intermittentErr.RetryPeriod).Wrap(err).Now() +// } +// ktesting.ExpectNoError(tCtx, err, "something failed") +// return value +// } +// tCtx.Eventually(cb).Should(gomega.Equal(42), "should be the answer to everything") +func Eventually[T any](tCtx TContext, cb func(TContext) T) gomega.AsyncAssertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Eventually(tCtx, func(ctx context.Context) (val T, err error) { + tCtx := WithContext(tCtx, ctx) + tCtx, finalize := WithError(tCtx, &err) + defer finalize() + tCtx = WithCancel(tCtx) + return cb(tCtx), nil + }) +} + +// Consistently wraps [gomega.Consistently] the same way as [Eventually] wraps +// [gomega.Eventually]. +func Consistently[T any](tCtx TContext, cb func(TContext) T) gomega.AsyncAssertion { + tCtx.Helper() + return gomega.NewWithT(tCtx).Consistently(tCtx, func(ctx context.Context) (val T, err error) { + tCtx := WithContext(tCtx, ctx) + tCtx, finalize := WithError(tCtx, &err) + defer finalize() + return cb(tCtx), nil + }) +} diff --git a/test/utils/ktesting/assert_test.go b/test/utils/ktesting/assert_test.go new file mode 100644 index 00000000000..5099eeb38d1 --- /dev/null +++ b/test/utils/ktesting/assert_test.go @@ -0,0 +1,194 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "fmt" + "regexp" + "testing" + "time" + + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" +) + +func TestAsync(t *testing.T) { + for name, tc := range map[string]struct { + cb func(TContext) + expectNoFail bool + expectError string + expectDuration time.Duration + }{ + "eventually-timeout": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) int { + // Canceling here is a nop. + tCtx.Cancel("testing") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +Expected + : 0 +to equal + : 1`, + }, + "eventually-final": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + gomega.StopTrying("final error").Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Told to stop trying after x.y s. +final error`, + }, + "eventually-error": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + tCtx.Fatal("some error") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +The function passed to Eventually returned the following error: + <*errors.joinError | 0xXXXX>: + some error + { + errs: [ + <*errors.errorString | 0xXXXX>{s: "some error"}, + ], + }`, + }, + "eventually-success": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + return 1.0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectNoFail: true, + expectError: ``, + }, + "eventually-retry": { + cb: func(tCtx TContext) { + Eventually(tCtx, func(tCtx TContext) float64 { + gomega.TryAgainAfter(time.Millisecond).Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out after x.y s. +told to try again after 1ms`, + }, + "consistently-timeout": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + // Canceling here is a nop. + tCtx.Cancel("testing") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Failed after x.y s. +Expected + : 0 +to equal + : 1`, + }, + "consistently-final": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + gomega.StopTrying("final error").Now() + tCtx.FailNow() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Told to stop trying after x.y s. +final error`, + }, + "consistently-error": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + tCtx.Fatal("some error") + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: 0, + expectError: `Failed after x.y s. +The function passed to Consistently returned the following error: + <*errors.joinError | 0xXXXX>: + some error + { + errs: [ + <*errors.errorString | 0xXXXX>{s: "some error"}, + ], + }`, + }, + "consistently-success": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + return 1.0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectNoFail: true, + expectError: ``, + }, + "consistently-retry": { + cb: func(tCtx TContext) { + Consistently(tCtx, func(tCtx TContext) float64 { + gomega.TryAgainAfter(time.Millisecond).Wrap(errors.New("intermittent error")).Now() + return 0 + }).WithTimeout(time.Second).Should(gomega.Equal(1.0)) + }, + expectDuration: time.Second, + expectError: `Timed out while waiting on TryAgainAfter after x.y s. +told to try again after 1ms: intermittent error`, + }, + } { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + tCtx := Init(t) + var err error + tCtx, finalize := WithError(tCtx, &err) + start := time.Now() + func() { + defer finalize() + tc.cb(tCtx) + }() + duration := time.Since(start) + assert.InDelta(t, tc.expectDuration.Seconds(), duration.Seconds(), 0.1, fmt.Sprintf("callback invocation duration %s", duration)) + assert.Equal(t, !tc.expectNoFail, tCtx.Failed(), "Failed()") + if tc.expectError == "" { + assert.NoError(t, err) + } else if assert.NotNil(t, err) { + t.Logf("Result:\n%s", err.Error()) + errMsg := err.Error() + errMsg = regexp.MustCompile(`[[:digit:]]+\.[[:digit:]]+s`).ReplaceAllString(errMsg, "x.y s") + errMsg = regexp.MustCompile(`0x[[:xdigit:]]+`).ReplaceAllString(errMsg, "0xXXXX") + assert.Equal(t, tc.expectError, errMsg) + } + }) + } +} diff --git a/test/utils/ktesting/clientcontext.go b/test/utils/ktesting/clientcontext.go new file mode 100644 index 00000000000..d07cbccf14a --- /dev/null +++ b/test/utils/ktesting/clientcontext.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "fmt" + + "github.com/onsi/gomega" + apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/klog/v2" +) + +// WithRESTConfig initializes all client-go clients with new clients +// created for the config. The current test name gets included in the UserAgent. +func WithRESTConfig(tCtx TContext, cfg *rest.Config) TContext { + cfg = rest.CopyConfig(cfg) + cfg.UserAgent = fmt.Sprintf("%s -- %s", rest.DefaultKubernetesUserAgent(), tCtx.Name()) + + cCtx := clientContext{ + TContext: tCtx, + restConfig: cfg, + client: clientset.NewForConfigOrDie(cfg), + dynamic: dynamic.NewForConfigOrDie(cfg), + apiextensions: apiextensions.NewForConfigOrDie(cfg), + } + + cachedDiscovery := memory.NewMemCacheClient(cCtx.client.Discovery()) + cCtx.restMapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscovery) + + return &cCtx +} + +// WithClients uses an existing config and clients. +func WithClients(tCtx TContext, cfg *rest.Config, mapper *restmapper.DeferredDiscoveryRESTMapper, client clientset.Interface, dynamic dynamic.Interface, apiextensions apiextensions.Interface) TContext { + return clientContext{ + TContext: tCtx, + restConfig: cfg, + restMapper: mapper, + client: client, + dynamic: dynamic, + apiextensions: apiextensions, + } +} + +type clientContext struct { + TContext + + restConfig *rest.Config + restMapper *restmapper.DeferredDiscoveryRESTMapper + client clientset.Interface + dynamic dynamic.Interface + apiextensions apiextensions.Interface +} + +func (cCtx clientContext) CleanupCtx(cb func(TContext)) { + cCtx.Helper() + cleanupCtx(cCtx, cb) +} + +func (cCtx clientContext) Expect(actual interface{}, extra ...interface{}) gomega.Assertion { + cCtx.Helper() + return expect(cCtx, actual, extra...) +} + +func (cCtx clientContext) ExpectNoError(err error, explain ...interface{}) { + cCtx.Helper() + expectNoError(cCtx, err, explain...) +} + +func (cCtx clientContext) Logger() klog.Logger { + return klog.FromContext(cCtx) +} + +func (cCtx clientContext) RESTConfig() *rest.Config { + if cCtx.restConfig == nil { + return nil + } + return rest.CopyConfig(cCtx.restConfig) +} + +func (cCtx clientContext) RESTMapper() *restmapper.DeferredDiscoveryRESTMapper { + return cCtx.restMapper +} + +func (cCtx clientContext) Client() clientset.Interface { + return cCtx.client +} + +func (cCtx clientContext) Dynamic() dynamic.Interface { + return cCtx.dynamic +} + +func (cCtx clientContext) APIExtensions() apiextensions.Interface { + return cCtx.apiextensions +} diff --git a/test/utils/ktesting/contexthelper.go b/test/utils/ktesting/contexthelper.go new file mode 100644 index 00000000000..28d2b245934 --- /dev/null +++ b/test/utils/ktesting/contexthelper.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "fmt" + "time" +) + +// cleanupErr creates a cause when canceling a context because the test has completed. +// It is a context.Canceled error. +func cleanupErr(testName string) error { + return canceledError(fmt.Sprintf("test %s is cleaning up", testName)) +} + +type canceledError string + +func (c canceledError) Error() string { return string(c) } + +func (c canceledError) Is(target error) bool { + return target == context.Canceled +} + +// withTimeout corresponds to [context.WithTimeout]. In contrast to +// [context.WithTimeout], it automatically cancels during test cleanup, provides +// the given cause when the deadline is reached, and its cancel function +// requires a cause. +func withTimeout(ctx context.Context, tb TB, timeout time.Duration, timeoutCause string) (context.Context, func(cause string)) { + tb.Helper() + + now := time.Now() + + cancelCtx, cancel := context.WithCancelCause(ctx) + after := time.NewTimer(timeout) + stopCtx, stop := context.WithCancel(ctx) // Only used internally, doesn't need a cause. + tb.Cleanup(func() { + cancel(cleanupErr(tb.Name())) + stop() + }) + go func() { + select { + case <-stopCtx.Done(): + after.Stop() + // No need to set a cause here. The cause or error of + // the parent context will be used. + case <-after.C: + cancel(canceledError(timeoutCause)) + } + }() + + // Determine which deadline is sooner: ours or that of our parent. + deadline := now.Add(timeout) + if parentDeadline, ok := ctx.Deadline(); ok { + if deadline.After(parentDeadline) { + deadline = parentDeadline + } + } + + // We always have a deadline. + return deadlineContext{Context: cancelCtx, deadline: deadline}, func(cause string) { + var cancelCause error + if cause != "" { + cancelCause = canceledError(cause) + } + cancel(cancelCause) + } +} + +type deadlineContext struct { + context.Context + deadline time.Time +} + +func (d deadlineContext) Deadline() (time.Time, bool) { + return d.deadline, true +} diff --git a/test/utils/ktesting/contexthelper_test.go b/test/utils/ktesting/contexthelper_test.go new file mode 100644 index 00000000000..548f99a975f --- /dev/null +++ b/test/utils/ktesting/contexthelper_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCleanupErr(t *testing.T) { + actual := cleanupErr(t.Name()) + if !errors.Is(actual, context.Canceled) { + t.Errorf("cleanupErr %T should be a %T", actual, context.Canceled) + } +} + +func TestCause(t *testing.T) { + timeoutCause := canceledError("I timed out") + parentCause := errors.New("parent canceled") + + t.Parallel() + for name, tt := range map[string]struct { + parentCtx context.Context + timeout time.Duration + sleep time.Duration + cancelCause string + expectErr, expectCause error + expectDeadline time.Duration + }{ + "nothing": { + parentCtx: context.Background(), + timeout: 5 * time.Millisecond, + sleep: time.Millisecond, + }, + "timeout": { + parentCtx: context.Background(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: canceledError(timeoutCause), + }, + "parent-canceled": { + parentCtx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: context.Canceled, + }, + "parent-cause": { + parentCtx: func() context.Context { + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(parentCause) + return ctx + }(), + timeout: time.Millisecond, + sleep: 5 * time.Millisecond, + expectErr: context.Canceled, + expectCause: parentCause, + }, + "deadline-no-parent": { + parentCtx: context.Background(), + timeout: time.Minute, + expectDeadline: time.Minute, + }, + "deadline-parent": { + parentCtx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + return ctx + }(), + timeout: 2 * time.Minute, + expectDeadline: time.Minute, + }, + "deadline-child": { + parentCtx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + return ctx + }(), + timeout: time.Minute, + expectDeadline: time.Minute, + }, + } { + tt := tt + t.Run(name, func(t *testing.T) { + ctx, cancel := withTimeout(tt.parentCtx, t, tt.timeout, timeoutCause.Error()) + if tt.cancelCause != "" { + cancel(tt.cancelCause) + } + if tt.expectDeadline != 0 { + actualDeadline, ok := ctx.Deadline() + if assert.True(t, ok, "should have had a deadline") { + assert.InDelta(t, time.Until(actualDeadline), tt.expectDeadline, float64(time.Second), "remaining time till Deadline()") + } + } + time.Sleep(tt.sleep) + actualErr := ctx.Err() + actualCause := context.Cause(ctx) + assert.Equal(t, tt.expectErr, actualErr, "ctx.Err()") + assert.Equal(t, tt.expectCause, actualCause, "context.Cause()") + + }) + } +} diff --git a/test/utils/ktesting/doc.go b/test/utils/ktesting/doc.go new file mode 100644 index 00000000000..3207f311bc8 --- /dev/null +++ b/test/utils/ktesting/doc.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ktesting is a wrapper around k8s.io/klog/v2/ktesting. In contrast +// to the klog package, this one is opinionated and tailored towards testing +// Kubernetes. +// +// Importing it +// - adds the -v command line flag +// - enables better dumping of complex datatypes +// - sets the default verbosity to 5 (can be changed with [SetDefaultVerbosity]) +// +// It also adds additional APIs and types for unit and integration tests +// which are too experimental for klog and/or are unrelated to logging. +// The ktesting package itself takes care of managing a test context +// with deadlines, timeouts, cancellation, and some common attributes +// as first-class members of the API. Sub-packages have additional APIs +// for propagating values via the context, implemented via [WithValue]. +package ktesting diff --git a/test/utils/ktesting/errorcontext.go b/test/utils/ktesting/errorcontext.go new file mode 100644 index 00000000000..a963b199e85 --- /dev/null +++ b/test/utils/ktesting/errorcontext.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "fmt" + "strings" + "sync" + + "k8s.io/klog/v2" +) + +// WithError creates a context where test failures are collected and stored in +// the provided error instance when the caller is done. Use it like this: +// +// func doSomething(tCtx ktesting.TContext) (finalErr error) { +// tCtx, finalize := WithError(tCtx, &finalErr) +// defer finalize() +// ... +// tCtx.Fatal("some failure") +// +// Any error already stored in the variable will get overwritten by finalize if +// there were test failures, otherwise the variable is left unchanged. +// If there were multiple test errors, then the error will wrap all of +// them with errors.Join. +// +// Test failures are not propagated to the parent context. +func WithError(tCtx TContext, err *error) (TContext, func()) { + eCtx := &errorContext{ + TContext: tCtx, + } + + return eCtx, func() { + // Recover has to be called in the deferred function. When called inside + // a function called by a deferred function (like finalize below), it + // returns nil. + if e := recover(); e != nil { + if _, ok := e.(fatalWithError); !ok { + // Not our own panic, pass it on instead of setting the error. + panic(e) + } + } + + eCtx.finalize(err) + } +} + +type errorContext struct { + TContext + + mutex sync.Mutex + errors []error + failed bool +} + +func (eCtx *errorContext) finalize(err *error) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + if !eCtx.failed { + return + } + + errs := eCtx.errors + if len(errs) == 0 { + errs = []error{errFailedWithNoExplanation} + } + *err = errors.Join(errs...) +} + +func (eCtx *errorContext) Error(args ...any) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + // Gomega adds a leading newline in https://github.com/onsi/gomega/blob/f804ac6ada8d36164ecae0513295de8affce1245/internal/gomega.go#L37 + // Let's strip that at start and end because ktesting will make errors + // stand out more with the "ERROR" prefix, so there's no need for additional + // line breaks. + eCtx.errors = append(eCtx.errors, errors.New(strings.TrimSpace(fmt.Sprintln(args...)))) + eCtx.failed = true +} + +func (eCtx *errorContext) Errorf(format string, args ...any) { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + eCtx.errors = append(eCtx.errors, errors.New(strings.TrimSpace(fmt.Sprintf(format, args...)))) + eCtx.failed = true +} + +func (eCtx *errorContext) Fail() { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + eCtx.failed = true +} + +func (eCtx *errorContext) FailNow() { + eCtx.Helper() + eCtx.Fail() + panic(failed) +} + +func (eCtx *errorContext) Failed() bool { + eCtx.mutex.Lock() + defer eCtx.mutex.Unlock() + + return eCtx.failed +} + +func (eCtx *errorContext) Fatal(args ...any) { + eCtx.Error(args...) + eCtx.FailNow() +} + +func (eCtx *errorContext) Fatalf(format string, args ...any) { + eCtx.Errorf(format, args...) + eCtx.FailNow() +} + +func (eCtx *errorContext) CleanupCtx(cb func(TContext)) { + eCtx.Helper() + cleanupCtx(eCtx, cb) +} + +func (eCtx *errorContext) Logger() klog.Logger { + return klog.FromContext(eCtx) +} + +// fatalWithError is the internal type that should never get propagated up. The +// only case where that can happen is when the developer forgot to call +// finalize via defer. The string explains that, in case that developers get to +// see it. +type fatalWithError string + +const failed = fatalWithError("WithError TContext encountered a fatal error, but the finalize function was not called via defer as it should have been.") + +var errFailedWithNoExplanation = errors.New("WithError context was marked as failed without recording an error") diff --git a/test/utils/ktesting/errorcontext_test.go b/test/utils/ktesting/errorcontext_test.go new file mode 100644 index 00000000000..24492d9353d --- /dev/null +++ b/test/utils/ktesting/errorcontext_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithError(t *testing.T) { + t.Run("panic", func(t *testing.T) { + assert.Panics(t, func() { + tCtx := Init(t) + var err error + _, finalize := WithError(tCtx, &err) + defer finalize() + + panic("pass me through") + }) + }) + + normalErr := errors.New("normal error") + + for name, tc := range map[string]struct { + cb func(TContext) + expectNoFail bool + expectError string + }{ + "none": { + cb: func(tCtx TContext) {}, + expectNoFail: true, + expectError: normalErr.Error(), + }, + "Error": { + cb: func(tCtx TContext) { + tCtx.Error("some error") + }, + expectError: "some error", + }, + "Errorf": { + cb: func(tCtx TContext) { + tCtx.Errorf("some %s", "error") + }, + expectError: "some error", + }, + "Fatal": { + cb: func(tCtx TContext) { + tCtx.Fatal("some error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "Fatalf": { + cb: func(tCtx TContext) { + tCtx.Fatalf("some %s", "error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "Fail": { + cb: func(tCtx TContext) { + tCtx.Fatalf("some %s", "error") + tCtx.Error("another error") + }, + expectError: "some error", + }, + "FailNow": { + cb: func(tCtx TContext) { + tCtx.FailNow() + tCtx.Error("another error") + }, + expectError: errFailedWithNoExplanation.Error(), + }, + "many": { + cb: func(tCtx TContext) { + tCtx.Error("first error") + tCtx.Error("second error") + }, + expectError: `first error +second error`, + }, + } { + t.Run(name, func(t *testing.T) { + tCtx := Init(t) + err := normalErr + tCtx, finalize := WithError(tCtx, &err) + func() { + defer finalize() + tc.cb(tCtx) + }() + + assert.Equal(t, !tc.expectNoFail, tCtx.Failed(), "Failed()") + if tc.expectError == "" { + assert.NoError(t, err) + } else if assert.NotNil(t, err) { + assert.Equal(t, tc.expectError, err.Error()) + } + }) + } +} diff --git a/test/utils/ktesting/examples/gomega/doc.go b/test/utils/ktesting/examples/gomega/doc.go new file mode 100644 index 00000000000..76312bb4eb0 --- /dev/null +++ b/test/utils/ktesting/examples/gomega/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package gomega diff --git a/test/utils/ktesting/examples/gomega/example_test.go b/test/utils/ktesting/examples/gomega/example_test.go new file mode 100644 index 00000000000..48150766f86 --- /dev/null +++ b/test/utils/ktesting/examples/gomega/example_test.go @@ -0,0 +1,44 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gomega + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "context" + "testing" + "time" + + "github.com/onsi/gomega" + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestGomega(t *testing.T) { + tCtx := ktesting.Init(t) + + gomega.NewWithT(tCtx).Eventually(tCtx, func(ctx context.Context) int { + // TODO: tCtx = ktesting.WithContext(tCtx, ctx) + // Or some dedicated tCtx.Eventually? + + return 42 + }).WithPolling(time.Second).Should(gomega.Equal(1)) +} diff --git a/test/utils/ktesting/examples/logging/doc.go b/test/utils/ktesting/examples/logging/doc.go new file mode 100644 index 00000000000..67085e2d5f0 --- /dev/null +++ b/test/utils/ktesting/examples/logging/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package logging diff --git a/test/utils/ktesting/examples/logging/example_test.go b/test/utils/ktesting/examples/logging/example_test.go new file mode 100644 index 00000000000..6202838e56e --- /dev/null +++ b/test/utils/ktesting/examples/logging/example_test.go @@ -0,0 +1,74 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "testing" + + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestError(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Error("some", "thing") +} + +func TestErrorf(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Errorf("some %s", "thing") +} + +func TestFatal(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Fatal("some", "thing") + tCtx.Log("not reached") +} + +func TestFatalf(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Fatalf("some %s", "thing") + tCtx.Log("not reached") +} + +func TestInfo(t *testing.T) { + tCtx := ktesting.Init(t) + tCtx.Log("hello via Log") + tCtx.Logger().Info("hello via Info") + tCtx.Error("some", "thing") +} + +func TestWithStep(t *testing.T) { + tCtx := ktesting.Init(t) + bake(ktesting.WithStep(tCtx, "bake cake")) +} + +func bake(tCtx ktesting.TContext) { + heatOven(ktesting.WithStep(tCtx, "set heat for baking")) +} + +func heatOven(tCtx ktesting.TContext) { + tCtx.Log("Log()") + tCtx.Logger().Info("Logger().Info()") + tCtx.Fatal("oven not found") +} diff --git a/test/utils/ktesting/examples/with_ktesting/doc.go b/test/utils/ktesting/examples/with_ktesting/doc.go new file mode 100644 index 00000000000..42b2a6b42fb --- /dev/null +++ b/test/utils/ktesting/examples/with_ktesting/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package withktesting diff --git a/test/utils/ktesting/examples/with_ktesting/example_test.go b/test/utils/ktesting/examples/with_ktesting/example_test.go new file mode 100644 index 00000000000..576f310c34c --- /dev/null +++ b/test/utils/ktesting/examples/with_ktesting/example_test.go @@ -0,0 +1,52 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package withktesting + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "context" + "testing" + "time" + + "k8s.io/kubernetes/test/utils/ktesting" +) + +func TestTimeout(t *testing.T) { + tCtx := ktesting.Init(t) + tmp := t.TempDir() + tCtx.Logf("Using %q as temporary directory.", tmp) + tCtx.Cleanup(func() { + t.Log("Cleaning up...") + }) + if deadline, ok := t.Deadline(); ok { + t.Logf("Will fail shortly before the test suite deadline at %s.", deadline) + } + select { + case <-time.After(1000 * time.Hour): + // This should not be reached. + tCtx.Log("Huh?! I shouldn't be that old.") + case <-tCtx.Done(): + // But this will before the test suite timeout. + tCtx.Errorf("need to stop: %v", context.Cause(tCtx)) + } +} diff --git a/test/utils/ktesting/examples/without_ktesting/doc.go b/test/utils/ktesting/examples/without_ktesting/doc.go new file mode 100644 index 00000000000..9b02e894aa3 --- /dev/null +++ b/test/utils/ktesting/examples/without_ktesting/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The tests will fail and therefore are excluded from normal "make test" via +// the "example" build tag. To run the tests and check the output, use "go test +// -tags example ." +package withoutktesting diff --git a/test/utils/ktesting/examples/without_ktesting/example_test.go b/test/utils/ktesting/examples/without_ktesting/example_test.go new file mode 100644 index 00000000000..d0bc37eac3b --- /dev/null +++ b/test/utils/ktesting/examples/without_ktesting/example_test.go @@ -0,0 +1,40 @@ +//go:build example +// +build example + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package withoutktesting + +// The tests below will fail and therefore are excluded from +// normal "make test" via the "example" build tag. To run +// the tests and check the output, use "go test -tags example ." + +import ( + "testing" + "time" +) + +func TestTimeout(t *testing.T) { + tmp := t.TempDir() + t.Logf("Using %q as temporary directory.", tmp) + t.Cleanup(func() { + t.Log("Cleaning up...") + }) + // This will not complete anytime soon... + t.Log("Please kill me.") + <-time.After(1000 * time.Hour) +} diff --git a/test/utils/ktesting/initoption/initoption.go b/test/utils/ktesting/initoption/initoption.go new file mode 100644 index 00000000000..cdac2be3950 --- /dev/null +++ b/test/utils/ktesting/initoption/initoption.go @@ -0,0 +1,30 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initoption + +import "k8s.io/kubernetes/test/utils/ktesting/internal" + +// InitOption is a functional option for Init and InitCtx. +type InitOption func(c *internal.InitConfig) + +// PerTestOutput controls whether a per-test logger gets +// set up by Init. Has no effect in InitCtx. +func PerTestOutput(enabled bool) InitOption { + return func(c *internal.InitConfig) { + c.PerTestOutput = enabled + } +} diff --git a/test/utils/ktesting/internal/config.go b/test/utils/ktesting/internal/config.go new file mode 100644 index 00000000000..22f7560aedf --- /dev/null +++ b/test/utils/ktesting/internal/config.go @@ -0,0 +1,21 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +type InitConfig struct { + PerTestOutput bool +} diff --git a/test/utils/ktesting/klogcontext.go b/test/utils/ktesting/klogcontext.go new file mode 100644 index 00000000000..08cd8e7e6be --- /dev/null +++ b/test/utils/ktesting/klogcontext.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktesting + +import ( + "fmt" + "strings" + "time" +) + +var timeNow = time.Now // Can be stubbed out for testing. + +// withKlogHeader creates a TB where the same "I