Merge pull request #108651 from andrewsykim/node-e2e-cred-provider

test/e2e_node: add kubelet credential provider tests

test/e2e_node/builder/build.go

@@ -35,6 +35,7 @@ var buildTargets = []string{
 	"test/e2e_node/e2e_node.test",
 	"vendor/github.com/onsi/ginkgo/ginkgo",
 	"cluster/gce/gci/mounter",
+	"test/e2e_node/plugins/gcp-credential-provider",
 }
 
 // BuildGo builds k8s binaries.

test/e2e_node/image_credential_provider.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2enode
+
+import (
+	"github.com/onsi/ginkgo"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+var _ = SIGDescribe("ImageCredentialProvider [Feature:KubeletCredentialProviders]", func() {
+	f := framework.NewDefaultFramework("image-credential-provider")
+	var podClient *framework.PodClient
+
+	ginkgo.BeforeEach(func() {
+		podClient = f.PodClient()
+	})
+
+	/*
+		Release: v1.24
+		Testname: Test kubelet image pull with external credential provider plugins
+		Description: Create Pod with an image from a private registry. This test assumes that the kubelet credential provider plugin is enabled for the registry hosting imageutils.AgnhostPrivate.
+	*/
+	ginkgo.It("should be able to create pod with image credentials fetched from external credential provider ", func() {
+		privateimage := imageutils.GetConfig(imageutils.AgnhostPrivate)
+		name := "pod-auth-image-" + string(uuid.NewUUID())
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+			Spec: v1.PodSpec{
+				Containers: []v1.Container{
+					{
+						Name:            "container-auth-image",
+						Image:           privateimage.GetE2EImage(),
+						ImagePullPolicy: v1.PullAlways,
+					},
+				},
+			},
+		}
+
+		// CreateSync tests that the Pod is running and ready
+		podClient.CreateSync(pod)
+	})
+})

test/e2e_node/plugins/gcp-credential-provider/README.md

@@ -0,0 +1,35 @@
+# GCP credential provider for e2e testing
+
+This package contains a barebones implementation of the [kubelet GCP credential
+provider](https://github.com/kubernetes/cloud-provider-gcp/tree/master/cmd/auth-provider-gcp)
+for testing purposes only. This plugin SHOULD NOT be used in production.
+
+This credential provider is installed and configured in the node e2e tests by:
+
+1. Building the gcp-credential-provider binary and including it in the test archive
+   uploaded to the GCE remote node.
+
+2. Writing the credential provider config into the temporary workspace consumed
+  by the kubelet. The contents of the config should be something like this:
+
+```yaml
+kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1alpha1
+providers:
+  - name: gcp-credential-provider
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+    matchImages:
+    - "gcr.io"
+    - "*.gcr.io"
+    - "container.cloud.google.com"
+    - "*.pkg.dev"
+    defaultCacheDuration: 1m`
+```
+
+3. Configuring the following additional flags on the kubelet:
+
+```
+--feature-gates=DisableKubeletCloudCredentialProviders=true,KubeletCredentialProviders=true
+--image-credential-provider-config=/tmp/node-e2e-123456/credential-provider.yaml
+--image-credential-provider-bin-dir=/tmp/node-e2e-12345
+```

test/e2e_node/plugins/gcp-credential-provider/main.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	credentialproviderv1alpha1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1"
+)
+
+const metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
+
+func main() {
+	if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil {
+		klog.Fatalf("failed to get credentials: %v", err)
+	}
+}
+
+func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
+	provider := &provider{
+		client: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+		tokenEndpoint: tokenEndpoint,
+	}
+
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	var authRequest credentialproviderv1alpha1.CredentialProviderRequest
+	err = json.Unmarshal(data, &authRequest)
+	if err != nil {
+		return err
+	}
+
+	auth, err := provider.Provide(authRequest.Image)
+	if err != nil {
+		return err
+	}
+
+	response := &credentialproviderv1alpha1.CredentialProviderResponse{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "CredentialProviderResponse",
+			APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1",
+		},
+		CacheKeyType: credentialproviderv1alpha1.RegistryPluginCacheKeyType,
+		Auth:         auth,
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		// The error from json.Marshal is intentionally not included so as to not leak credentials into the logs
+		return errors.New("error marshaling response")
+	}
+
+	return nil
+}

test/e2e_node/plugins/gcp-credential-provider/main_test.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeTokenServer struct {
+	token string
+}
+
+func (f *fakeTokenServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	fmt.Fprintf(w, `{"access_token": "%s"}`, f.token)
+}
+
+func Test_getCredentials(t *testing.T) {
+	server := httptest.NewServer(&fakeTokenServer{token: "abc123"})
+	defer server.Close()
+
+	in := bytes.NewBuffer([]byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1alpha1","image":"gcr.io/foobar"}`))
+	out := bytes.NewBuffer(nil)
+
+	err := getCredentials(server.URL, in, out)
+	if err != nil {
+		t.Fatalf("unexpected error running getCredentials: %v", err)
+	}
+
+	expected := `{"kind":"CredentialProviderResponse","apiVersion":"credentialprovider.kubelet.k8s.io/v1alpha1","cacheKeyType":"Registry","auth":{"*.gcr.io":{"username":"_token","password":"abc123"},"*.pkg.dev":{"username":"_token","password":"abc123"},"container.cloud.google.com":{"username":"_token","password":"abc123"},"gcr.io":{"username":"_token","password":"abc123"}}}
+`
+
+	if out.String() != expected {
+		t.Logf("actual response: %v", out)
+		t.Logf("expected response: %v", expected)
+		t.Errorf("unexpected credential provider response")
+	}
+}

test/e2e_node/plugins/gcp-credential-provider/provider.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Originally copied from pkg/credentialproviders/gcp
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	credentialproviderv1alpha1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1"
+)
+
+const (
+	maxReadLength = 10 * 1 << 20 // 10MB
+)
+
+var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"}
+
+// HTTPError wraps a non-StatusOK error code as an error.
+type HTTPError struct {
+	StatusCode int
+	URL        string
+}
+
+var _ error = &HTTPError{}
+
+// Error implements error
+func (h *HTTPError) Error() string {
+	return fmt.Sprintf("http status code: %d while fetching url %s",
+		h.StatusCode, h.URL)
+}
+
+// TokenBlob is used to decode the JSON blob containing an access token
+// that is returned by GCE metadata.
+type TokenBlob struct {
+	AccessToken string `json:"access_token"`
+}
+
+type provider struct {
+	client        *http.Client
+	tokenEndpoint string
+}
+
+func (p *provider) Provide(image string) (map[string]credentialproviderv1alpha1.AuthConfig, error) {
+	cfg := map[string]credentialproviderv1alpha1.AuthConfig{}
+
+	tokenJSONBlob, err := readURL(p.tokenEndpoint, p.client)
+	if err != nil {
+		return cfg, err
+	}
+
+	var parsedBlob TokenBlob
+	if err := json.Unmarshal(tokenJSONBlob, &parsedBlob); err != nil {
+		return cfg, err
+	}
+
+	authConfig := credentialproviderv1alpha1.AuthConfig{
+		Username: "_token",
+		Password: parsedBlob.AccessToken,
+	}
+
+	// Add our entry for each of the supported container registry URLs
+	for _, k := range containerRegistryUrls {
+		cfg[k] = authConfig
+	}
+	return cfg, nil
+}
+
+func readURL(url string, client *http.Client) (body []byte, err error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header = http.Header{
+		"Metadata-Flavor": []string{"Google"},
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, &HTTPError{
+			StatusCode: resp.StatusCode,
+			URL:        url,
+		}
+	}
+
+	limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength}
+	contents, err := ioutil.ReadAll(limitedReader)
+	if err != nil {
+		return nil, err
+	}
+
+	if limitedReader.N <= 0 {
+		return nil, errors.New("the read limit is reached")
+	}
+
+	return contents, nil
+}

test/e2e_node/remote/node_e2e.go

@@ -60,7 +60,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 	}
 
 	// Copy binaries
-	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter"}
+	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"}
 	for _, bin := range requiredBins {
 		source := filepath.Join(buildOutputDir, bin)
 		if _, err := os.Stat(source); err != nil {
@@ -102,6 +102,16 @@ func prependMemcgNotificationFlag(args string) string {
 	return "--kubelet-flags=--kernel-memcg-notification=true " + args
 }
 
+// prependGCPCredentialProviderFlag prepends the flags for enabling
+// a credential provider plugin.
+func prependGCPCredentialProviderFlag(args, workspace string) string {
+	credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml")
+	featureGateFlag := "--kubelet-flags=--feature-gates=DisableKubeletCloudCredentialProviders=true,KubeletCredentialProviders=true"
+	configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig)
+	binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace)
+	return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args)
+}
+
 // osSpecificActions takes OS specific actions required for the node tests
 func osSpecificActions(args, host, workspace string) (string, error) {
 	output, err := getOSDistribution(host)
@@ -114,6 +124,7 @@ func osSpecificActions(args, host, workspace string) (string, error) {
 		return args, setKubeletSELinuxLabels(host, workspace)
 	case strings.Contains(output, "gci"), strings.Contains(output, "cos"):
 		args = prependMemcgNotificationFlag(args)
+		args = prependGCPCredentialProviderFlag(args, workspace)
 		return prependCOSMounterFlag(args, host, workspace)
 	case strings.Contains(output, "ubuntu"):
 		return prependMemcgNotificationFlag(args), nil
@@ -166,6 +177,11 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr
 		return "", err
 	}
 
+	// Install the kubelet credential provider plugin
+	if err := configureCredentialProvider(host, workspace); err != nil {
+		return "", err
+	}
+
 	// Kill any running node processes
 	cleanupNodeProcesses(host)
 

test/e2e_node/remote/utils.go

@@ -48,6 +48,18 @@ const cniConfig = `{
 }
 `
 
+const credentialProviderConfig = `kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1alpha1
+providers:
+  - name: gcp-credential-provider
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+    matchImages:
+    - "gcr.io"
+    - "*.gcr.io"
+    - "container.cloud.google.com"
+    - "*.pkg.dev"
+    defaultCacheDuration: 1m`
+
 // Install the cni plugin and add basic bridge configuration to the
 // configuration directory.
 func setupCNI(host, workspace string) error {
@@ -76,6 +88,19 @@ func setupCNI(host, workspace string) error {
 	return nil
 }
 
+func configureCredentialProvider(host, workspace string) error {
+	klog.V(2).Infof("Configuring kubelet credential provider on %q", host)
+
+	cmd := getSSHCommand(" ; ",
+		fmt.Sprintf("echo %s > %s", quote(credentialProviderConfig), filepath.Join(workspace, "credential-provider.yaml")),
+	)
+	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
+		return fmt.Errorf("failed to write credential provider configuration on %q: %v output: %q", host, err, output)
+	}
+
+	return nil
+}
+
 // configureFirewall configures iptable firewall rules.
 func configureFirewall(host string) error {
 	klog.V(2).Infof("Configure iptables firewall rules on %q", host)