From 1dc5fb0190afed98041292cea09d0eeadafc0591 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Wed, 19 Aug 2020 20:27:55 -0400 Subject: [PATCH 1/8] kubelet: respect dockershim exec timeout Signed-off-by: Andrew Sy Kim --- pkg/kubelet/dockershim/exec.go | 56 +++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index 31e00bda18b..e0fb5fcebc4 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -21,6 +21,7 @@ package dockershim import ( "fmt" "io" + "strings" "time" dockertypes "github.com/docker/docker/api/types" @@ -30,6 +31,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" + utilexec "k8s.io/utils/exec" ) // ExecHandler knows how to execute a command in a running Docker container. @@ -110,29 +112,49 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container return err } + var execTimeout <-chan time.Time + if timeout > 0 { + execTimeout = time.After(timeout) + } else { + // skip exec timeout if provided timeout is 0 + execTimeout = make(chan time.Time, 1) + } + ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() count := 0 for { - inspect, err2 := client.InspectExec(execObj.ID) - if err2 != nil { - return err2 - } - if !inspect.Running { - if inspect.ExitCode != 0 { - err = &dockerExitError{inspect} + select { + case <-execTimeout: + // If exec timed out, return utilexec.CodeExitError with an exit status as expected + // from prober for failed probes. + // TODO: utilexec should have a TimedoutError type and we should return it here once available. + return utilexec.CodeExitError{ + Err: fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), + Code: 1, // exit code here doesn't really matter, as long as it's not 0 + } + // need to use "default" here instead of <-ticker.C, otherwise we delay the initial InspectExec by 2 seconds. + default: + inspect, inspectErr := client.InspectExec(execObj.ID) + if inspectErr != nil { + return inspectErr } - break - } - count++ - if count == 5 { - klog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) - break - } + if !inspect.Running { + if inspect.ExitCode != 0 { + return &dockerExitError{inspect} + } - <-ticker.C + return nil + } + + count++ + if count == 5 { + klog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) + return nil + } + + <-ticker.C + } } - - return err } From 2887e707f871d7c7db95ad277d3fad4b67652f97 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Thu, 20 Aug 2020 20:44:52 -0400 Subject: [PATCH 2/8] kubelet: respect probe exec timeout from CRI by returning utilexec.CodeExitError This fixes a bug where the exec timeouts are not respected with containerd Exec prober expects a utilexec.CodeExitError on failed probes, otherwise the prober returns 'Unknown' and a non-nil error which the kubelet throws away. As a temporary fix, ExecSync as part of the CRI remote runtime should return utilexec.CodeExitError when the grpc error code is DeadlineContextExceeded. This ensure the exec prober registers exec timeouts as real probe failures to the kubelet. We should also add a TimededError type to k8s.io/utils/exec since it doesn't really make sense to use CodeExitError for exec time outs. Signed-off-by: Andrew Sy Kim --- pkg/kubelet/cri/remote/BUILD | 2 ++ pkg/kubelet/cri/remote/remote_runtime.go | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/kubelet/cri/remote/BUILD b/pkg/kubelet/cri/remote/BUILD index 5ea0f5db56b..ea28193df68 100644 --- a/pkg/kubelet/cri/remote/BUILD +++ b/pkg/kubelet/cri/remote/BUILD @@ -21,6 +21,8 @@ go_library( "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], diff --git a/pkg/kubelet/cri/remote/remote_runtime.go b/pkg/kubelet/cri/remote/remote_runtime.go index d51d1876222..80ad3aad193 100644 --- a/pkg/kubelet/cri/remote/remote_runtime.go +++ b/pkg/kubelet/cri/remote/remote_runtime.go @@ -24,6 +24,8 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "k8s.io/klog/v2" "k8s.io/component-base/logs/logreduction" @@ -387,6 +389,17 @@ func (r *remoteRuntimeService) ExecSync(containerID string, cmd []string, timeou resp, err := r.runtimeClient.ExecSync(ctx, req) if err != nil { klog.Errorf("ExecSync %s '%s' from runtime service failed: %v", containerID, strings.Join(cmd, " "), err) + + // If exec timed out, return utilexec.CodeExitError with an exit status as expected + // from prober for failed probes. + // TODO: utilexec should have a TimedoutError type and we should return it here once available. + if status.Code(err) == codes.DeadlineExceeded { + err = utilexec.CodeExitError{ + Err: fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), + Code: 1, // exit code here doesn't really matter, as long as it's not 0 + } + } + return nil, nil, err } From 733862ed7fea203138ab69282f6e26f6f0903ffe Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Thu, 20 Aug 2020 16:07:12 -0400 Subject: [PATCH 3/8] kubelet: stop skipping docker exec livenessprobe timeout test Signed-off-by: Andrew Sy Kim --- test/e2e/common/container_probe.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/e2e/common/container_probe.go b/test/e2e/common/container_probe.go index 6fb540f4b9e..31967154827 100644 --- a/test/e2e/common/container_probe.go +++ b/test/e2e/common/container_probe.go @@ -32,7 +32,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" testutils "k8s.io/kubernetes/test/utils" "github.com/onsi/ginkgo" @@ -213,8 +212,6 @@ var _ = framework.KubeDescribe("Probing container", func() { Description: A Pod is created with liveness probe with a Exec action on the Pod. If the liveness probe call does not return within the timeout specified, liveness probe MUST restart the Pod. */ ginkgo.It("should be restarted with a docker exec liveness probe with timeout ", func() { - // TODO: enable this test once the default exec handler supports timeout. - e2eskipper.Skipf("The default exec handler, dockertools.NativeExecHandler, does not support timeouts due to a limitation in the Docker Remote API") cmd := []string{"/bin/sh", "-c", "sleep 600"} livenessProbe := &v1.Probe{ Handler: execHandler([]string{"/bin/sh", "-c", "sleep 10"}), From 81ad32e3a4005be16a4403a5f1edef342a16113e Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Sat, 22 Aug 2020 14:42:50 -0400 Subject: [PATCH 4/8] kubelet: add e2e test for exec readiness probe timeout Signed-off-by: Andrew Sy Kim --- test/e2e/common/container_probe.go | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/e2e/common/container_probe.go b/test/e2e/common/container_probe.go index 31967154827..d85cb6d1a3d 100644 --- a/test/e2e/common/container_probe.go +++ b/test/e2e/common/container_probe.go @@ -223,6 +223,23 @@ var _ = framework.KubeDescribe("Probing container", func() { RunLivenessTest(f, pod, 1, defaultObservationTimeout) }) + /* + Release: v1.20 + Testname: Pod readiness probe, docker exec, not ready + Description: A Pod is created with readiness probe with a Exec action on the Pod. If the readiness probe call does not return within the timeout specified, readiness probe MUST not be Ready. + */ + ginkgo.It("should not be ready with a docker exec readiness probe timeout ", func() { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + readinessProbe := &v1.Probe{ + Handler: execHandler([]string{"/bin/sh", "-c", "sleep 10"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + FailureThreshold: 1, + } + pod := busyBoxPodSpec(readinessProbe, nil, cmd) + runReadinessFailTest(f, pod, time.Minute) + }) + /* Release: v1.14 Testname: Pod http liveness probe, redirected to a local address @@ -622,3 +639,35 @@ func RunLivenessTest(f *framework.Framework, pod *v1.Pod, expectNumRestarts int, ns, pod.Name, expectNumRestarts, observedRestarts) } } + +func runReadinessFailTest(f *framework.Framework, pod *v1.Pod, notReadyUntil time.Duration) { + podClient := f.PodClient() + ns := f.Namespace.Name + gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty()) + + // At the end of the test, clean up by removing the pod. + defer func() { + ginkgo.By("deleting the pod") + podClient.Delete(context.TODO(), pod.Name, *metav1.NewDeleteOptions(0)) + }() + ginkgo.By(fmt.Sprintf("Creating pod %s in namespace %s", pod.Name, ns)) + podClient.Create(pod) + + // Wait until the pod is not pending. (Here we need to check for something other than + // 'Pending', since when failures occur, we go to 'Terminated' which can cause indefinite blocking.) + framework.ExpectNoError(e2epod.WaitForPodNotPending(f.ClientSet, ns, pod.Name), + fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) + framework.Logf("Started pod %s in namespace %s", pod.Name, ns) + + // Wait for the not ready state to be true for notReadyUntil duration + deadline := time.Now().Add(notReadyUntil) + for start := time.Now(); time.Now().Before(deadline); time.Sleep(2 * time.Second) { + // poll for Not Ready + if podutil.IsPodReady(pod) { + framework.Failf("pod %s/%s - expected to be not ready", ns, pod.Name) + } + + framework.Logf("pod %s/%s is not ready (%v elapsed)", + ns, pod.Name, time.Since(start)) + } +} From 8ef299552b5331561cbdf20792c9e6583da23d6f Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Sat, 22 Aug 2020 15:15:59 -0400 Subject: [PATCH 5/8] kubelet: allow dockershim exec timeouts to be longer than 10s Signed-off-by: Andrew Sy Kim --- pkg/kubelet/dockershim/exec.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index e0fb5fcebc4..8e53597c8db 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -117,7 +117,7 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container execTimeout = time.After(timeout) } else { // skip exec timeout if provided timeout is 0 - execTimeout = make(chan time.Time, 1) + execTimeout = nil } ticker := time.NewTicker(2 * time.Second) @@ -148,10 +148,14 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container return nil } - count++ - if count == 5 { - klog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) - return nil + // Only limit the amount of InspectExec calls if the exec timeout was not set. + // When a timeout is not set, we stop polling the exec session after 5 attempts and allow the process to continue running. + if execTimeout == nil { + count++ + if count == 5 { + klog.Errorf("Exec session %s in container %s terminated but process still running!", execObj.ID, container.ID) + return nil + } } <-ticker.C From 0b6ef2123371fc88643ba1cae24c22dd566d64ad Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Tue, 13 Oct 2020 22:44:23 -0400 Subject: [PATCH 6/8] features: add ExecProbeTimeout feature gate Signed-off-by: Andrew Sy Kim --- pkg/features/kube_features.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 53c8bdf19c3..8bcf72ecdbc 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -668,6 +668,14 @@ const ( // alpha: v1.21 // LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode" + + // owner: @andrewsykim @SergeyKanzhelev + // GA: v1.20 + // + // Ensure kubelet respects exec probe timeouts. Feature gate exists in-case existing workloads + // may depend on old behavior where exec probe timeouts were ignored. + // Lock to default in v1.21 and remove in v1.22. + ExecProbeTimeout featuregate.Feature = "ExecProbeTimeout" ) func init() { @@ -769,6 +777,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS RootCAConfigMap: {Default: true, PreRelease: featuregate.Beta}, SizeMemoryBackedVolumes: {Default: false, PreRelease: featuregate.Alpha}, LoadBalancerIPMode: {Default: false, PreRelease: featuregate.Alpha}, + ExecProbeTimeout: {Default: true, PreRelease: featuregate.GA}, // lock to default in v1.21 and remove in v1.22 // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: From 4bb30c3b0e9167eb85b1e3b0c9a3bab3b351a3e2 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Sun, 6 Sep 2020 13:52:49 -0400 Subject: [PATCH 7/8] kubelet: add feature gate check for exec probe timeouts This change also involves adding a custom error type for probe timeouts so that the kubelet exec prober can distinguish between failed probes that have exited or probes that have timed out. Signed-off-by: Andrew Sy Kim --- pkg/kubelet/cri/.import-restrictions | 1 + pkg/kubelet/cri/remote/BUILD | 1 + pkg/kubelet/cri/remote/remote_runtime.go | 10 ++--- pkg/kubelet/dockershim/BUILD | 3 ++ pkg/kubelet/dockershim/exec.go | 15 +++----- pkg/probe/exec/BUILD | 7 +++- pkg/probe/exec/errors.go | 47 ++++++++++++++++++++++++ pkg/probe/exec/exec.go | 12 ++++++ 8 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 pkg/probe/exec/errors.go diff --git a/pkg/kubelet/cri/.import-restrictions b/pkg/kubelet/cri/.import-restrictions index 10215ff9251..ae9260b9bd1 100644 --- a/pkg/kubelet/cri/.import-restrictions +++ b/pkg/kubelet/cri/.import-restrictions @@ -3,3 +3,4 @@ rules: - selectorRegexp: k8s[.]io/kubernetes allowedPrefixes: - k8s.io/kubernetes/pkg/kubelet/cri + - k8s.io/kubernetes/pkg/probe/exec diff --git a/pkg/kubelet/cri/remote/BUILD b/pkg/kubelet/cri/remote/BUILD index ea28193df68..d6cd6b7cf5b 100644 --- a/pkg/kubelet/cri/remote/BUILD +++ b/pkg/kubelet/cri/remote/BUILD @@ -17,6 +17,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/cri/remote", deps = [ "//pkg/kubelet/cri/remote/util:go_default_library", + "//pkg/probe/exec:go_default_library", "//staging/src/k8s.io/component-base/logs/logreduction:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", diff --git a/pkg/kubelet/cri/remote/remote_runtime.go b/pkg/kubelet/cri/remote/remote_runtime.go index 80ad3aad193..87b41bbf16f 100644 --- a/pkg/kubelet/cri/remote/remote_runtime.go +++ b/pkg/kubelet/cri/remote/remote_runtime.go @@ -32,6 +32,7 @@ import ( internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/cri/remote/util" + "k8s.io/kubernetes/pkg/probe/exec" utilexec "k8s.io/utils/exec" ) @@ -390,14 +391,9 @@ func (r *remoteRuntimeService) ExecSync(containerID string, cmd []string, timeou if err != nil { klog.Errorf("ExecSync %s '%s' from runtime service failed: %v", containerID, strings.Join(cmd, " "), err) - // If exec timed out, return utilexec.CodeExitError with an exit status as expected - // from prober for failed probes. - // TODO: utilexec should have a TimedoutError type and we should return it here once available. + // interpret DeadlineExceeded gRPC errors as timedout probes if status.Code(err) == codes.DeadlineExceeded { - err = utilexec.CodeExitError{ - Err: fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), - Code: 1, // exit code here doesn't really matter, as long as it's not 0 - } + err = exec.NewTimeoutError(fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), timeout) } return nil, nil, err diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 06e3c04e599..4bd4958e11e 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -37,6 +37,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/credentialprovider:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/checksum:go_default_library", @@ -55,11 +56,13 @@ go_library( "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/cache:go_default_library", "//pkg/kubelet/util/ioutils:go_default_library", + "//pkg/probe/exec:go_default_library", "//pkg/util/parsers:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//vendor/github.com/armon/circbuf:go_default_library", diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index 8e53597c8db..d79cff31d97 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -27,11 +27,13 @@ import ( dockertypes "github.com/docker/docker/api/types" "k8s.io/klog/v2" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/probe/exec" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" - utilexec "k8s.io/utils/exec" ) // ExecHandler knows how to execute a command in a running Docker container. @@ -112,8 +114,9 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container return err } + // if ExecProbeTimeout feature gate is disabled, preserve existing behavior to ignore exec timeouts var execTimeout <-chan time.Time - if timeout > 0 { + if timeout > 0 && utilfeature.DefaultFeatureGate.Enabled(features.ExecProbeTimeout) { execTimeout = time.After(timeout) } else { // skip exec timeout if provided timeout is 0 @@ -126,13 +129,7 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container for { select { case <-execTimeout: - // If exec timed out, return utilexec.CodeExitError with an exit status as expected - // from prober for failed probes. - // TODO: utilexec should have a TimedoutError type and we should return it here once available. - return utilexec.CodeExitError{ - Err: fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), - Code: 1, // exit code here doesn't really matter, as long as it's not 0 - } + return exec.NewTimeoutError(fmt.Errorf("command %q timed out", strings.Join(cmd, " ")), timeout) // need to use "default" here instead of <-ticker.C, otherwise we delay the initial InspectExec by 2 seconds. default: inspect, inspectErr := client.InspectExec(execObj.ID) diff --git a/pkg/probe/exec/BUILD b/pkg/probe/exec/BUILD index ce381191610..80186df8ca9 100644 --- a/pkg/probe/exec/BUILD +++ b/pkg/probe/exec/BUILD @@ -8,11 +8,16 @@ load( go_library( name = "go_default_library", - srcs = ["exec.go"], + srcs = [ + "errors.go", + "exec.go", + ], importpath = "k8s.io/kubernetes/pkg/probe/exec", deps = [ + "//pkg/features:go_default_library", "//pkg/kubelet/util/ioutils:go_default_library", "//pkg/probe:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], diff --git a/pkg/probe/exec/errors.go b/pkg/probe/exec/errors.go new file mode 100644 index 00000000000..6659a94ac7b --- /dev/null +++ b/pkg/probe/exec/errors.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exec + +import ( + "time" +) + +// NewTimeoutError returns a new TimeoutError. +func NewTimeoutError(err error, timeout time.Duration) *TimeoutError { + return &TimeoutError{ + err: err, + timeout: timeout, + } +} + +// TimeoutError is an error returned on exec probe timeouts. It should be returned by CRI implementations +// in order for the exec prober to interpret exec timeouts as failed probes. +// TODO: this error type can likely be removed when we support CRI errors. +type TimeoutError struct { + err error + timeout time.Duration +} + +// Error returns the error string. +func (t *TimeoutError) Error() string { + return t.err.Error() +} + +// Timeout returns the timeout duration of the exec probe. +func (t *TimeoutError) Timeout() time.Duration { + return t.timeout +} diff --git a/pkg/probe/exec/exec.go b/pkg/probe/exec/exec.go index 9aa894672ad..164d21bd112 100644 --- a/pkg/probe/exec/exec.go +++ b/pkg/probe/exec/exec.go @@ -19,6 +19,8 @@ package exec import ( "bytes" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/util/ioutils" "k8s.io/kubernetes/pkg/probe" @@ -66,6 +68,16 @@ func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) { } return probe.Failure, string(data), nil } + + timeoutErr, ok := err.(*TimeoutError) + if ok { + if utilfeature.DefaultFeatureGate.Enabled(features.ExecProbeTimeout) { + return probe.Failure, string(data), nil + } + + klog.Warningf("Exec probe timed out after %s but ExecProbeTimeout feature gate was disabled", timeoutErr.Timeout()) + } + return probe.Unknown, "", err } return probe.Success, string(data), nil From af40d18b6792cee723f1f25bd5bfd2db79de0fd7 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Fri, 16 Oct 2020 14:30:21 -0400 Subject: [PATCH 8/8] kubelet: move pkg/kubelet/cri/.import-restrictions to pkg/kubelet/cri/streaming Based on the comments in this file, it seems like these import restrictions were originally meant for the kubelet CRI streaming package. This commit moves the import restrictions to pkg/kubelet/cri/streaming so that pkg/kubelet/cri can import internal packages like pkg/probe/exec Signed-off-by: Andrew Sy Kim --- pkg/kubelet/cri/{ => streaming}/.import-restrictions | 1 - 1 file changed, 1 deletion(-) rename pkg/kubelet/cri/{ => streaming}/.import-restrictions (80%) diff --git a/pkg/kubelet/cri/.import-restrictions b/pkg/kubelet/cri/streaming/.import-restrictions similarity index 80% rename from pkg/kubelet/cri/.import-restrictions rename to pkg/kubelet/cri/streaming/.import-restrictions index ae9260b9bd1..10215ff9251 100644 --- a/pkg/kubelet/cri/.import-restrictions +++ b/pkg/kubelet/cri/streaming/.import-restrictions @@ -3,4 +3,3 @@ rules: - selectorRegexp: k8s[.]io/kubernetes allowedPrefixes: - k8s.io/kubernetes/pkg/kubelet/cri - - k8s.io/kubernetes/pkg/probe/exec