add e2e test with the gcp-credential-provider test plugin

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

test/e2e/common/node/image_credential_provider.go

@@ -24,6 +24,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/kubernetes/test/e2e/feature"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -34,10 +35,12 @@ import (
 var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProviders, func() {
 	f := framework.NewDefaultFramework("image-credential-provider")
 	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
+	var serviceAccountClient typedcorev1.ServiceAccountInterface
 	var podClient *e2epod.PodClient
 
 	ginkgo.BeforeEach(func() {
 		podClient = e2epod.NewPodClient(f)
+		serviceAccountClient = f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name)
 	})
 
 	/*
@@ -48,11 +51,28 @@ var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProvider
 	ginkgo.It("should be able to create pod with image credentials fetched from external credential provider ", func(ctx context.Context) {
 		privateimage := imageutils.GetConfig(imageutils.AgnhostPrivate)
 		name := "pod-auth-image-" + string(uuid.NewUUID())
+
+		// The service account is required to exist for the credential provider plugin that's configured to use service account token.
+		serviceAccount := &v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-service-account",
+				// these annotations are validated by the test gcp-credential-provider-with-sa plugin
+				// that runs in service account token mode.
+				Annotations: map[string]string{
+					"domain.io/identity-id":   "123456",
+					"domain.io/identity-type": "serviceaccount",
+				},
+			},
+		}
+		_, err := serviceAccountClient.Create(ctx, serviceAccount, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
 		pod := &v1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: name,
 			},
 			Spec: v1.PodSpec{
+				ServiceAccountName: "test-service-account",
 				Containers: []v1.Container{
 					{
 						Name:            "container-auth-image",

test/e2e_node/plugins/gcp-credential-provider/main.go

@@ -17,19 +17,29 @@ limitations under the License.
 package main
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"reflect"
+	"strings"
 	"time"
 
+	"gopkg.in/go-jose/go-jose.v2/jwt"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
 )
 
-const metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
+const (
+	metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
+
+	pluginModeEnvVar = "PLUGIN_MODE"
+)
 
 func main() {
 	if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil {
@@ -56,6 +66,40 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
 		return err
 	}
 
+	pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount"
+	if pluginUsingServiceAccount {
+		if len(authRequest.ServiceAccountToken) == 0 {
+			return errors.New("service account token is empty")
+		}
+		expectedAnnotations := map[string]string{
+			"domain.io/identity-id":   "123456",
+			"domain.io/identity-type": "serviceaccount",
+		}
+		if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) {
+			return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations)
+		}
+		// The service account token is not actually used for authentication by this test plugin.
+		// We extract the claims from the token to validate the audience.
+		// This is solely for testing assertions and is not an actual security layer.
+		// Post validation in this block, we proceed with the default flow for fetching credentials.
+		c, err := getClaims(authRequest.ServiceAccountToken)
+		if err != nil {
+			return err
+		}
+		// The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience
+		// in CredentialProviderConfig.
+		if len(c.Audience) != 1 || c.Audience[0] != "test-audience" {
+			return fmt.Errorf("unexpected audience: %v", c.Audience)
+		}
+	} else {
+		if len(authRequest.ServiceAccountToken) > 0 {
+			return errors.New("service account token is not expected")
+		}
+		if len(authRequest.ServiceAccountAnnotations) > 0 {
+			return errors.New("service account annotations are not expected")
+		}
+	}
+
 	auth, err := provider.Provide(authRequest.Image)
 	if err != nil {
 		return err
@@ -70,6 +114,10 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
 		Auth:         auth,
 	}
 
+	if pluginUsingServiceAccount {
+		response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType
+	}
+
 	if err := json.NewEncoder(w).Encode(response); err != nil {
 		// The error from json.Marshal is intentionally not included so as to not leak credentials into the logs
 		return errors.New("error marshaling response")
@@ -77,3 +125,55 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
 
 	return nil
 }
+
+// getClaims is used to extract claims from the service account token when the plugin is running in service account mode
+// This is solely for testing assertions and is not an actual security layer.
+// We get claims and validate the audience of the token (audience in the token matches the audience configured
+// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig).
+func getClaims(tokenData string) (claims, error) {
+	if strings.HasPrefix(strings.TrimSpace(tokenData), "{") {
+		return claims{}, errors.New("token is not a JWS")
+	}
+	parts := strings.Split(tokenData, ".")
+	if len(parts) != 3 {
+		return claims{}, errors.New("token is not a JWS")
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return claims{}, fmt.Errorf("error decoding token payload: %w", err)
+	}
+
+	var c claims
+	d := json.NewDecoder(strings.NewReader(string(payload)))
+	d.DisallowUnknownFields()
+	if err := d.Decode(&c); err != nil {
+		return claims{}, fmt.Errorf("error decoding token payload: %w", err)
+	}
+
+	return c, nil
+}
+
+type claims struct {
+	jwt.Claims
+	privateClaims
+}
+
+// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67
+
+type privateClaims struct {
+	Kubernetes kubernetes `json:"kubernetes.io,omitempty"`
+}
+
+type kubernetes struct {
+	Namespace string           `json:"namespace,omitempty"`
+	Svcacct   ref              `json:"serviceaccount,omitempty"`
+	Pod       *ref             `json:"pod,omitempty"`
+	Secret    *ref             `json:"secret,omitempty"`
+	Node      *ref             `json:"node,omitempty"`
+	WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"`
+}
+
+type ref struct {
+	Name string `json:"name,omitempty"`
+	UID  string `json:"uid,omitempty"`
+}

test/e2e_node/remote/node_e2e.go

@@ -72,6 +72,21 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 		}
 	}
 
+	// create a symlink of gcp-credential-provider binary to use for testing
+	// service account token for credential providers.
+	// feature-gate: KubeletServiceAccountTokenForCredentialProviders=true
+	binary := "gcp-credential-provider" // Use relative path instead of full path
+	symlink := filepath.Join(tardir, "gcp-credential-provider-with-sa")
+	if _, err := os.Lstat(symlink); err == nil {
+		if err := os.Remove(symlink); err != nil {
+			return fmt.Errorf("failed to remove symlink %q: %w", symlink, err)
+		}
+	}
+	klog.V(2).Infof("Creating symlink %s -> %s", symlink, binary)
+	if err := os.Symlink(binary, symlink); err != nil {
+		return fmt.Errorf("failed to create symlink %q: %w", symlink, err)
+	}
+
 	if systemSpecName != "" {
 		// Copy system spec file
 		source := filepath.Join(rootDir, system.SystemSpecPath, systemSpecName+".yaml")
@@ -97,9 +112,10 @@ func prependMemcgNotificationFlag(args string) string {
 // a credential provider plugin.
 func prependCredentialProviderFlag(args, workspace string) string {
 	credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml")
+	featureGateFlag := "--kubelet-flags=--feature-gates=KubeletServiceAccountTokenForCredentialProviders=true"
 	configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig)
 	binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace)
-	return fmt.Sprintf("%s %s %s", configFlag, binFlag, args)
+	return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args)
 }
 
 // osSpecificActions takes OS specific actions required for the node tests

test/e2e_node/remote/utils.go

@@ -53,14 +53,31 @@ const cniConfig = `{
 const credentialGCPProviderConfig = `kind: CredentialProviderConfig
 apiVersion: kubelet.config.k8s.io/v1
 providers:
-  - name: gcp-credential-provider
+ - name: gcp-credential-provider
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
+   apiVersion: credentialprovider.kubelet.k8s.io/v1
-    matchImages:
+   matchImages:
-    - "gcr.io"
+   - "gcr.io"
-    - "*.gcr.io"
+   - "*.gcr.io"
-    - "container.cloud.google.com"
+   - "container.cloud.google.com"
-    - "*.pkg.dev"
+   - "*.pkg.dev"
-    defaultCacheDuration: 1m`
+   defaultCacheDuration: 1m
+ - name: gcp-credential-provider-with-sa
+   apiVersion: credentialprovider.kubelet.k8s.io/v1
+   matchImages:
+   - "gcr.io"
+   - "*.gcr.io"
+   - "container.cloud.google.com"
+   - "*.pkg.dev"
+   defaultCacheDuration: 1m
+   tokenAttributes:
+     serviceAccountTokenAudience: test-audience
+     requireServiceAccount: true
+     requiredServiceAccountAnnotationKeys:
+     - "domain.io/identity-id"
+     - "domain.io/identity-type"
+   env:
+   - name: PLUGIN_MODE
+     value: "serviceaccount"`
 
 const credentialAWSProviderConfig = `kind: CredentialProviderConfig
 apiVersion: kubelet.config.k8s.io/v1