diff --git a/plugin/pkg/client/auth/exec/metrics.go b/plugin/pkg/client/auth/exec/metrics.go
index 3a2cc251..51210975 100644
--- a/plugin/pkg/client/auth/exec/metrics.go
+++ b/plugin/pkg/client/auth/exec/metrics.go
@@ -18,6 +18,7 @@ package exec
 
 import (
 	"errors"
+	"io/fs"
 	"os/exec"
 	"reflect"
 	"sync"
@@ -92,6 +93,7 @@ func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) {
 func incrementCallsMetric(err error) {
 	execExitError := &exec.ExitError{}
 	execError := &exec.Error{}
+	pathError := &fs.PathError{}
 	switch {
 	case err == nil: // Binary execution succeeded.
 		metrics.ExecPluginCalls.Increment(successExitCode, noError)
@@ -99,7 +101,7 @@ func incrementCallsMetric(err error) {
 	case errors.As(err, &execExitError): // Binary execution failed (see "os/exec".Cmd.Run()).
 		metrics.ExecPluginCalls.Increment(execExitError.ExitCode(), pluginExecutionError)
 
-	case errors.As(err, &execError): // Binary does not exist (see exec.Error).
+	case errors.As(err, &execError), errors.As(err, &pathError): // Binary does not exist (see exec.Error, fs.PathError).
 		metrics.ExecPluginCalls.Increment(failureExitCode, pluginNotFoundError)
 
 	default: // We don't know about this error type.
diff --git a/plugin/pkg/client/auth/exec/metrics_test.go b/plugin/pkg/client/auth/exec/metrics_test.go
index 80d84480..4488df59 100644
--- a/plugin/pkg/client/auth/exec/metrics_test.go
+++ b/plugin/pkg/client/auth/exec/metrics_test.go
@@ -18,6 +18,7 @@ package exec
 
 import (
 	"fmt"
+	"io"
 	"testing"
 	"time"
 
@@ -147,6 +148,7 @@ func TestCallsMetric(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		a.stderr = io.Discard
 
 		// Run refresh creds twice so that our test validates that the metrics are set correctly twice
 		// in a row with the same authenticator.
@@ -172,7 +174,7 @@ func TestCallsMetric(t *testing.T) {
 	// metric values.
 	refreshCreds := func(command string) {
 		c := api.ExecConfig{
-			Command:         "does not exist",
+			Command:         command,
 			APIVersion:      "client.authentication.k8s.io/v1beta1",
 			InteractiveMode: api.IfAvailableExecInteractiveMode,
 		}
@@ -180,6 +182,7 @@ func TestCallsMetric(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		a.stderr = io.Discard
 		if err := a.refreshCredsLocked(&clientauthentication.Response{}); err == nil {
 			t.Fatal("expected the authenticator to fail because the plugin does not exist")
 		}