diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
index 00787c71f1c..6789605231b 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
@@ -48,7 +48,8 @@ const (
 	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
 	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
 	kmsTransformerPrefixV1       = "k8s:enc:kms:v1:"
-	kmsPluginHealthzTTL          = 3 * time.Second
+	kmsPluginHealthzNegativeTTL  = 3 * time.Second
+	kmsPluginHealthzPositiveTTL  = 20 * time.Second
 )
 
 type kmsPluginHealthzResponse struct {
@@ -58,6 +59,7 @@ type kmsPluginHealthzResponse struct {
 
 type kmsPluginProbe struct {
 	name string
+	ttl  time.Duration
 	envelope.Service
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
@@ -112,6 +114,7 @@ func getKMSPluginProbes(reader io.Reader) ([]*kmsPluginProbe, error) {
 
 				result = append(result, &kmsPluginProbe{
 					name:         p.KMS.Name,
+					ttl:          kmsPluginHealthzNegativeTTL,
 					Service:      s,
 					l:            &sync.Mutex{},
 					lastResponse: &kmsPluginHealthzResponse{},
@@ -128,22 +131,25 @@ func (h *kmsPluginProbe) Check() error {
 	h.l.Lock()
 	defer h.l.Unlock()
 
-	if (time.Since(h.lastResponse.received)) < kmsPluginHealthzTTL {
+	if (time.Since(h.lastResponse.received)) < h.ttl {
 		return h.lastResponse.err
 	}
 
 	p, err := h.Service.Encrypt([]byte("ping"))
 	if err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
 		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
 	}
 
 	if _, err := h.Service.Decrypt(p); err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
 		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
 	}
 
 	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	h.ttl = kmsPluginHealthzPositiveTTL
 	return nil
 }
 
diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go
index 53124fbcddf..6ef07a8a740 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go
@@ -19,9 +19,11 @@ package encryptionconfig
 import (
 	"bytes"
 	"encoding/base64"
+	"errors"
 	"io"
 	"io/ioutil"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
@@ -61,19 +63,31 @@ func mustConfigReader(t *testing.T, path string) io.Reader {
 // testEnvelopeService is a mock envelope service which can be used to simulate remote Envelope services
 // for testing of the envelope transformer with other transformers.
 type testEnvelopeService struct {
+	err error
 }
 
 func (t *testEnvelopeService) Decrypt(data []byte) ([]byte, error) {
+	if t.err != nil {
+		return nil, t.err
+	}
 	return base64.StdEncoding.DecodeString(string(data))
 }
 
 func (t *testEnvelopeService) Encrypt(data []byte) ([]byte, error) {
+	if t.err != nil {
+		return nil, t.err
+	}
 	return []byte(base64.StdEncoding.EncodeToString(data)), nil
 }
 
 // The factory method to create mock envelope service.
 func newMockEnvelopeService(endpoint string, timeout time.Duration) (envelope.Service, error) {
-	return &testEnvelopeService{}, nil
+	return &testEnvelopeService{nil}, nil
+}
+
+// The factory method to create mock envelope service which always returns error.
+func newMockErrorEnvelopeService(endpoint string, timeout time.Duration) (envelope.Service, error) {
+	return &testEnvelopeService{errors.New("test")}, nil
 }
 
 func TestLegacyConfig(t *testing.T) {
@@ -261,6 +275,49 @@ func TestKMSPluginHealthz(t *testing.T) {
 	}
 }
 
+func TestKMSPluginHealthzTTL(t *testing.T) {
+	service, _ := newMockEnvelopeService("unix:///tmp/testprovider.sock", 3*time.Second)
+	errService, _ := newMockErrorEnvelopeService("unix:///tmp/testprovider.sock", 3*time.Second)
+
+	testCases := []struct {
+		desc    string
+		probe   *kmsPluginProbe
+		wantTTL time.Duration
+	}{
+		{
+			desc: "kms provider in good state",
+			probe: &kmsPluginProbe{
+				name:         "test",
+				ttl:          kmsPluginHealthzNegativeTTL,
+				Service:      service,
+				l:            &sync.Mutex{},
+				lastResponse: &kmsPluginHealthzResponse{},
+			},
+			wantTTL: kmsPluginHealthzPositiveTTL,
+		},
+		{
+			desc: "kms provider in bad state",
+			probe: &kmsPluginProbe{
+				name:         "test",
+				ttl:          kmsPluginHealthzPositiveTTL,
+				Service:      errService,
+				l:            &sync.Mutex{},
+				lastResponse: &kmsPluginHealthzResponse{},
+			},
+			wantTTL: kmsPluginHealthzNegativeTTL,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt.probe.Check()
+			if tt.probe.ttl != tt.wantTTL {
+				t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl)
+			}
+		})
+	}
+}
+
 // As long as got and want contain envelope.Service we will return true.
 // If got has an envelope.Service and want does note (or vice versa) this will return false.
 func serviceComparer(_, _ envelope.Service) bool {