exec credential provider: add install hint

This commit adds the ability for users to specify an install hint for
their exec credential provider binary.

In the exec credential provider workflow, if the exec credential binary
does not exist, then the user will see some sort of ugly

  exec: exec: "does-not-exist": executable file not found in $PATH

error message.  If some user downloads a kubeconfig from somewhere, they
may not know that kubectl is trying to use a binary to obtain
credentials to auth to the API, and scratch their head when they see
this error message.  Furthermore, even if a user does know that their
kubeconfig is trying to run a binary, they might not know how to obtain
the binary.  This install hint seeks to ease the above 2 user pains.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>

Kubernetes-commit: 94e2065df2eef3b198942efb156ef6e27abcc6f9

plugin/pkg/client/auth/exec/exec.go

@@ -29,6 +29,7 @@ import (
 	"os"
 	"os/exec"
 	"reflect"
+	"strings"
 	"sync"
 	"time"
 
@@ -38,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/clock"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/pkg/apis/clientauthentication"
 	"k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
@@ -51,6 +53,12 @@ import (
 
 const execInfoEnv = "KUBERNETES_EXEC_INFO"
 const onRotateListWarningLength = 1000
+const installHintVerboseHelp = `
+
+It looks like you are trying to use a client-go credential plugin that is not installed.
+
+To learn more about this feature, consult the documentation available at:
+      https://kubernetes.io/docs/reference/access-authn-authz/authentication/#client-go-credential-plugins`
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
@@ -108,6 +116,44 @@ func (c *cache) put(s string, a *Authenticator) *Authenticator {
 	return a
 }
 
+// sometimes rate limits how often a function f() is called. Specifically, Do()
+// will run the provided function f() up to threshold times every interval
+// duration.
+type sometimes struct {
+	threshold int
+	interval  time.Duration
+
+	clock clock.Clock
+	mu    sync.Mutex
+
+	count  int       // times we have called f() in this window
+	window time.Time // beginning of current window of length interval
+}
+
+func (s *sometimes) Do(f func()) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	now := s.clock.Now()
+	if s.window.IsZero() {
+		s.window = now
+	}
+
+	// If we are no longer in our saved time window, then we get to reset our run
+	// count back to 0 and start increasing towards the threshold again.
+	if inWindow := now.Sub(s.window) < s.interval; !inWindow {
+		s.window = now
+		s.count = 0
+	}
+
+	// If we have not run the function more than threshold times in this current
+	// time window, we get to run it now!
+	if underThreshold := s.count < s.threshold; underThreshold {
+		s.count++
+		f()
+	}
+}
+
 // GetAuthenticator returns an exec-based plugin for providing client credentials.
 func GetAuthenticator(config *api.ExecConfig) (*Authenticator, error) {
 	return newAuthenticator(globalCache, config)
@@ -129,6 +175,13 @@ func newAuthenticator(c *cache, config *api.ExecConfig) (*Authenticator, error)
 		args:  config.Args,
 		group: gv,
 
+		installHint: config.InstallHint,
+		sometimes: &sometimes{
+			threshold: 10,
+			interval:  time.Hour,
+			clock:     clock.RealClock{},
+		},
+
 		stdin:       os.Stdin,
 		stderr:      os.Stderr,
 		interactive: terminal.IsTerminal(int(os.Stdout.Fd())),
@@ -152,6 +205,12 @@ type Authenticator struct {
 	group schema.GroupVersion
 	env   []string
 
+	// Used to avoid log spew by rate limiting install hint printing. We didn't do
+	// this by interval based rate limiting alone since that way may have prevented
+	// the install hint from showing up for kubectl users.
+	sometimes   *sometimes
+	installHint string
+
 	// Stubbable for testing
 	stdin       io.Reader
 	stderr      io.Writer
@@ -323,7 +382,7 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 	}
 
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("exec: %v", err)
+		return a.wrapCmdRunErrorLocked(err)
 	}
 
 	_, gvk, err := codecs.UniversalDecoder(a.group).Decode(stdout.Bytes(), nil, cred)
@@ -394,3 +453,35 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 	expirationMetrics.set(a, expiry)
 	return nil
 }
+
+// wrapCmdRunErrorLocked pulls out the code to construct a helpful error message
+// for when the exec plugin's binary fails to Run().
+//
+// It must be called while holding the Authenticator's mutex.
+func (a *Authenticator) wrapCmdRunErrorLocked(err error) error {
+	switch err.(type) {
+	case *exec.Error: // Binary does not exist (see exec.Error).
+		builder := strings.Builder{}
+		fmt.Fprintf(&builder, "exec: executable %s not found", a.cmd)
+
+		a.sometimes.Do(func() {
+			fmt.Fprint(&builder, installHintVerboseHelp)
+			if a.installHint != "" {
+				fmt.Fprintf(&builder, "\n\n%s", a.installHint)
+			}
+		})
+
+		return errors.New(builder.String())
+
+	case *exec.ExitError: // Binary execution failed (see exec.Cmd.Run()).
+		e := err.(*exec.ExitError)
+		return fmt.Errorf(
+			"exec: executable %s failed with exit code %d",
+			a.cmd,
+			e.ProcessState.ExitCode(),
+		)
+
+	default:
+		return fmt.Errorf("exec: %v", err)
+	}
+}

plugin/pkg/client/auth/exec/exec_test.go

@@ -32,12 +32,14 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/client-go/pkg/apis/clientauthentication"
 	"k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/transport"
@@ -168,15 +170,17 @@ func compJSON(t *testing.T, got, want []byte) {
 
 func TestRefreshCreds(t *testing.T) {
 	tests := []struct {
-		name        string
-		config      api.ExecConfig
-		output      string
-		interactive bool
-		response    *clientauthentication.Response
-		wantInput   string
-		wantCreds   credentials
-		wantExpiry  time.Time
-		wantErr     bool
+		name          string
+		config        api.ExecConfig
+		exitCode      int
+		output        string
+		interactive   bool
+		response      *clientauthentication.Response
+		wantInput     string
+		wantCreds     credentials
+		wantExpiry    time.Time
+		wantErr       bool
+		wantErrSubstr string
 	}{
 		{
 			name: "basic-request",
@@ -450,17 +454,42 @@ func TestRefreshCreds(t *testing.T) {
 			}`,
 			wantErr: true,
 		},
+		{
+			name: "unknown-binary",
+			config: api.ExecConfig{
+				APIVersion:  "client.authentication.k8s.io/v1beta1",
+				Command:     "does not exist",
+				InstallHint: "some install hint",
+			},
+			wantErr:       true,
+			wantErrSubstr: "some install hint",
+		},
+		{
+			name: "binary-fails",
+			config: api.ExecConfig{
+				APIVersion: "client.authentication.k8s.io/v1beta1",
+			},
+			exitCode:      73,
+			wantErr:       true,
+			wantErrSubstr: "73",
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			c := test.config
 
-			c.Command = "./testdata/test-plugin.sh"
-			c.Env = append(c.Env, api.ExecEnvVar{
-				Name:  "TEST_OUTPUT",
-				Value: test.output,
-			})
+			if c.Command == "" {
+				c.Command = "./testdata/test-plugin.sh"
+				c.Env = append(c.Env, api.ExecEnvVar{
+					Name:  "TEST_OUTPUT",
+					Value: test.output,
+				})
+				c.Env = append(c.Env, api.ExecEnvVar{
+					Name:  "TEST_EXIT_CODE",
+					Value: strconv.Itoa(test.exitCode),
+				})
+			}
 
 			a, err := newAuthenticator(newCache(), &c)
 			if err != nil {
@@ -475,6 +504,8 @@ func TestRefreshCreds(t *testing.T) {
 			if err := a.refreshCredsLocked(test.response); err != nil {
 				if !test.wantErr {
 					t.Errorf("get token %v", err)
+				} else if !strings.Contains(err.Error(), test.wantErrSubstr) {
+					t.Errorf("expected error with substring '%v' got '%v'", test.wantErrSubstr, err.Error())
 				}
 				return
 			}
@@ -763,6 +794,75 @@ func TestConcurrentUpdateTransportConfig(t *testing.T) {
 	time.Sleep(2 * time.Second)
 }
 
+func TestInstallHintRateLimit(t *testing.T) {
+	tests := []struct {
+		name string
+
+		threshold int
+		interval  time.Duration
+
+		calls          int
+		perCallAdvance time.Duration
+
+		wantInstallHint int
+	}{
+		{
+			name:            "print-up-to-threshold",
+			threshold:       2,
+			interval:        time.Second,
+			calls:           10,
+			wantInstallHint: 2,
+		},
+		{
+			name:            "after-interval-threshold-resets",
+			threshold:       2,
+			interval:        time.Second * 5,
+			calls:           10,
+			perCallAdvance:  time.Second,
+			wantInstallHint: 4,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c := api.ExecConfig{
+				Command:     "does not exist",
+				APIVersion:  "client.authentication.k8s.io/v1alpha1",
+				InstallHint: "some install hint",
+			}
+			a, err := newAuthenticator(newCache(), &c)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			a.sometimes.threshold = test.threshold
+			a.sometimes.interval = test.interval
+
+			clock := clock.NewFakeClock(time.Now())
+			a.sometimes.clock = clock
+
+			count := 0
+			for i := 0; i < test.calls; i++ {
+				err := a.refreshCredsLocked(&clientauthentication.Response{})
+				if strings.Contains(err.Error(), c.InstallHint) {
+					count++
+				}
+
+				clock.SetTime(clock.Now().Add(test.perCallAdvance))
+			}
+
+			if test.wantInstallHint != count {
+				t.Errorf(
+					"%s: expected install hint %d times got %d",
+					test.name,
+					test.wantInstallHint,
+					count,
+				)
+			}
+		})
+	}
+}
+
 // genClientCert generates an x509 certificate for testing. Certificate and key
 // are returned in PEM encoding. The generated cert expires in 24 hours.
 func genClientCert(t *testing.T) ([]byte, []byte) {

plugin/pkg/client/auth/exec/testdata/test-plugin.sh

@@ -16,3 +16,4 @@
 
 >&2 echo "$KUBERNETES_EXEC_INFO"
 echo "$TEST_OUTPUT"
+exit "${TEST_EXIT_CODE:-0}"