diff --git a/test/e2e/common/node/image_credential_provider.go b/test/e2e/common/node/image_credential_provider.go index 52f2d82a9a5..e6c43afb39d 100644 --- a/test/e2e/common/node/image_credential_provider.go +++ b/test/e2e/common/node/image_credential_provider.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -34,10 +35,12 @@ import ( var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProviders, func() { f := framework.NewDefaultFramework("image-credential-provider") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var serviceAccountClient typedcorev1.ServiceAccountInterface var podClient *e2epod.PodClient ginkgo.BeforeEach(func() { podClient = e2epod.NewPodClient(f) + serviceAccountClient = f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name) }) /* @@ -48,11 +51,28 @@ var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProvider ginkgo.It("should be able to create pod with image credentials fetched from external credential provider ", func(ctx context.Context) { privateimage := imageutils.GetConfig(imageutils.AgnhostPrivate) name := "pod-auth-image-" + string(uuid.NewUUID()) + + // The service account is required to exist for the credential provider plugin that's configured to use service account token. + serviceAccount := &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service-account", + // these annotations are validated by the test gcp-credential-provider-with-sa plugin + // that runs in service account token mode. + Annotations: map[string]string{ + "domain.io/identity-id": "123456", + "domain.io/identity-type": "serviceaccount", + }, + }, + } + _, err := serviceAccountClient.Create(ctx, serviceAccount, metav1.CreateOptions{}) + framework.ExpectNoError(err) + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.PodSpec{ + ServiceAccountName: "test-service-account", Containers: []v1.Container{ { Name: "container-auth-image", diff --git a/test/e2e_node/plugins/gcp-credential-provider/main.go b/test/e2e_node/plugins/gcp-credential-provider/main.go index 58cdbda3636..b8be2718182 100644 --- a/test/e2e_node/plugins/gcp-credential-provider/main.go +++ b/test/e2e_node/plugins/gcp-credential-provider/main.go @@ -17,19 +17,29 @@ limitations under the License. package main import ( + "encoding/base64" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "reflect" + "strings" "time" + "gopkg.in/go-jose/go-jose.v2/jwt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" ) -const metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token" +const ( + metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token" + + pluginModeEnvVar = "PLUGIN_MODE" +) func main() { if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil { @@ -56,6 +66,40 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error { return err } + pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount" + if pluginUsingServiceAccount { + if len(authRequest.ServiceAccountToken) == 0 { + return errors.New("service account token is empty") + } + expectedAnnotations := map[string]string{ + "domain.io/identity-id": "123456", + "domain.io/identity-type": "serviceaccount", + } + if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) { + return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations) + } + // The service account token is not actually used for authentication by this test plugin. + // We extract the claims from the token to validate the audience. + // This is solely for testing assertions and is not an actual security layer. + // Post validation in this block, we proceed with the default flow for fetching credentials. + c, err := getClaims(authRequest.ServiceAccountToken) + if err != nil { + return err + } + // The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience + // in CredentialProviderConfig. + if len(c.Audience) != 1 || c.Audience[0] != "test-audience" { + return fmt.Errorf("unexpected audience: %v", c.Audience) + } + } else { + if len(authRequest.ServiceAccountToken) > 0 { + return errors.New("service account token is not expected") + } + if len(authRequest.ServiceAccountAnnotations) > 0 { + return errors.New("service account annotations are not expected") + } + } + auth, err := provider.Provide(authRequest.Image) if err != nil { return err @@ -70,6 +114,10 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error { Auth: auth, } + if pluginUsingServiceAccount { + response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType + } + if err := json.NewEncoder(w).Encode(response); err != nil { // The error from json.Marshal is intentionally not included so as to not leak credentials into the logs return errors.New("error marshaling response") @@ -77,3 +125,55 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error { return nil } + +// getClaims is used to extract claims from the service account token when the plugin is running in service account mode +// This is solely for testing assertions and is not an actual security layer. +// We get claims and validate the audience of the token (audience in the token matches the audience configured +// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig). +func getClaims(tokenData string) (claims, error) { + if strings.HasPrefix(strings.TrimSpace(tokenData), "{") { + return claims{}, errors.New("token is not a JWS") + } + parts := strings.Split(tokenData, ".") + if len(parts) != 3 { + return claims{}, errors.New("token is not a JWS") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + var c claims + d := json.NewDecoder(strings.NewReader(string(payload))) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + return c, nil +} + +type claims struct { + jwt.Claims + privateClaims +} + +// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67 + +type privateClaims struct { + Kubernetes kubernetes `json:"kubernetes.io,omitempty"` +} + +type kubernetes struct { + Namespace string `json:"namespace,omitempty"` + Svcacct ref `json:"serviceaccount,omitempty"` + Pod *ref `json:"pod,omitempty"` + Secret *ref `json:"secret,omitempty"` + Node *ref `json:"node,omitempty"` + WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"` +} + +type ref struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` +} diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 25c8ea92a2d..98fd4eb334b 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -72,6 +72,21 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } } + // create a symlink of gcp-credential-provider binary to use for testing + // service account token for credential providers. + // feature-gate: KubeletServiceAccountTokenForCredentialProviders=true + binary := "gcp-credential-provider" // Use relative path instead of full path + symlink := filepath.Join(tardir, "gcp-credential-provider-with-sa") + if _, err := os.Lstat(symlink); err == nil { + if err := os.Remove(symlink); err != nil { + return fmt.Errorf("failed to remove symlink %q: %w", symlink, err) + } + } + klog.V(2).Infof("Creating symlink %s -> %s", symlink, binary) + if err := os.Symlink(binary, symlink); err != nil { + return fmt.Errorf("failed to create symlink %q: %w", symlink, err) + } + if systemSpecName != "" { // Copy system spec file source := filepath.Join(rootDir, system.SystemSpecPath, systemSpecName+".yaml") @@ -97,9 +112,10 @@ func prependMemcgNotificationFlag(args string) string { // a credential provider plugin. func prependCredentialProviderFlag(args, workspace string) string { credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml") + featureGateFlag := "--kubelet-flags=--feature-gates=KubeletServiceAccountTokenForCredentialProviders=true" configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig) binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace) - return fmt.Sprintf("%s %s %s", configFlag, binFlag, args) + return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args) } // osSpecificActions takes OS specific actions required for the node tests diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go index 39ad30cb686..c3fab32099b 100644 --- a/test/e2e_node/remote/utils.go +++ b/test/e2e_node/remote/utils.go @@ -53,14 +53,31 @@ const cniConfig = `{ const credentialGCPProviderConfig = `kind: CredentialProviderConfig apiVersion: kubelet.config.k8s.io/v1 providers: - - name: gcp-credential-provider - apiVersion: credentialprovider.kubelet.k8s.io/v1 - matchImages: - - "gcr.io" - - "*.gcr.io" - - "container.cloud.google.com" - - "*.pkg.dev" - defaultCacheDuration: 1m` + - name: gcp-credential-provider + apiVersion: credentialprovider.kubelet.k8s.io/v1 + matchImages: + - "gcr.io" + - "*.gcr.io" + - "container.cloud.google.com" + - "*.pkg.dev" + defaultCacheDuration: 1m + - name: gcp-credential-provider-with-sa + apiVersion: credentialprovider.kubelet.k8s.io/v1 + matchImages: + - "gcr.io" + - "*.gcr.io" + - "container.cloud.google.com" + - "*.pkg.dev" + defaultCacheDuration: 1m + tokenAttributes: + serviceAccountTokenAudience: test-audience + requireServiceAccount: true + requiredServiceAccountAnnotationKeys: + - "domain.io/identity-id" + - "domain.io/identity-type" + env: + - name: PLUGIN_MODE + value: "serviceaccount"` const credentialAWSProviderConfig = `kind: CredentialProviderConfig apiVersion: kubelet.config.k8s.io/v1