Merge pull request #114625 from Divya063/feature-private-image-registry

[E2E] Add support for pulling images from private registry

test/e2e/framework/framework.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
+	v1svc "k8s.io/client-go/applyconfigurations/core/v1"
 	"k8s.io/client-go/discovery"
 	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
@@ -55,6 +56,7 @@ import (
 const (
 	// DefaultNamespaceDeletionTimeout is timeout duration for waiting for a namespace deletion.
 	DefaultNamespaceDeletionTimeout = 5 * time.Minute
+	defaultServiceAccountName       = "default"
 )
 
 var (
@@ -104,6 +106,7 @@ type Framework struct {
 	ScalesGetter scaleclient.ScalesGetter
 
 	SkipNamespaceCreation            bool            // Whether to skip creating a namespace
+	SkipSecretCreation               bool            // Whether to skip creating secret for a test
 	Namespace                        *v1.Namespace   // Every test has at least one namespace unless creation is skipped
 	namespacesToDelete               []*v1.Namespace // Some tests have more than one.
 	NamespaceDeletionTimeout         time.Duration
@@ -262,6 +265,7 @@ func (f *Framework) BeforeEach(ctx context.Context) {
 		} else {
 			Logf("Skipping waiting for service account")
 		}
+
 		f.UniqueName = f.Namespace.GetName()
 	} else {
 		// not guaranteed to be unique, but very likely
@@ -455,9 +459,48 @@ func (f *Framework) CreateNamespace(ctx context.Context, baseName string, labels
 	// fail to create serviceAccount in it.
 	f.AddNamespacesToDelete(ns)
 
+	if TestContext.E2EDockerConfigFile != "" && !f.SkipSecretCreation {
+		// With the Secret created, the default service account (in the new namespace)
+		// is patched with the secret and can then be referenced by all the pods spawned by E2E process, and repository authentication should be successful.
+		secret, err := f.createSecretFromDockerConfig(ctx, ns.Name)
+		if err != nil {
+			return ns, fmt.Errorf("failed to create secret from docker config file: %v", err)
+		}
+
+		serviceAccountClient := f.ClientSet.CoreV1().ServiceAccounts(ns.Name)
+		serviceAccountConfig := v1svc.ServiceAccount(defaultServiceAccountName, ns.Name)
+		serviceAccountConfig.ImagePullSecrets = append(serviceAccountConfig.ImagePullSecrets, v1svc.LocalObjectReferenceApplyConfiguration{Name: &secret.Name})
+
+		svc, err := serviceAccountClient.Apply(ctx, serviceAccountConfig, metav1.ApplyOptions{FieldManager: "e2e-framework"})
+		if err != nil {
+			return ns, fmt.Errorf("failed to patch imagePullSecret [%s] to service account [%s]: %v", secret.Name, svc.Name, err)
+		}
+
+	}
+
 	return ns, err
 }
 
+// createSecretFromDockerConfig creates a secret using the private image registry credentials.
+// The credentials are provided by --e2e-docker-config-file flag.
+func (f *Framework) createSecretFromDockerConfig(ctx context.Context, namespace string) (*v1.Secret, error) {
+	contents, err := os.ReadFile(TestContext.E2EDockerConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading docker config file: %v", err)
+	}
+
+	secretObject := &v1.Secret{
+		Data: map[string][]byte{v1.DockerConfigJsonKey: contents},
+		Type: v1.SecretTypeDockerConfigJson,
+	}
+	secretObject.GenerateName = "registry-cred"
+	Logf("create image pull secret %s", secretObject.Name)
+
+	secret, err := f.ClientSet.CoreV1().Secrets(namespace).Create(ctx, secretObject, metav1.CreateOptions{})
+
+	return secret, err
+}
+
 // RecordFlakeIfError records flakeness info if error happens.
 // NOTE: This function is not used at any places yet, but we are in progress for https://github.com/kubernetes/kubernetes/issues/66239 which requires this. Please don't remove this.
 func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) {

test/e2e/framework/test_context.go

@@ -189,6 +189,13 @@ type TestContextType struct {
 	// DockerConfigFile is a file that contains credentials which can be used to pull images from certain private registries, needed for a test.
 	DockerConfigFile string
 
+	// E2EDockerConfigFile is a docker credentials configuration file used which contains authorization token that can be used to pull images from certain private registries provided by the users.
+	// For more details refer https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#log-in-to-docker-hub
+	E2EDockerConfigFile string
+
+	// KubeTestRepoConfigFile is a yaml file used for overriding registries for test images.
+	KubeTestRepoList string
+
 	// SnapshotControllerPodName is the name used for identifying the snapshot controller pod.
 	SnapshotControllerPodName string
 
@@ -359,7 +366,10 @@ func RegisterCommonFlags(flags *flag.FlagSet) {
 
 	flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.")
 	flags.StringVar(&TestContext.SpecSummaryOutput, "spec-dump", "", "The file to dump all ginkgo.SpecSummary to after tests run. If empty, no objects are saved/printed.")
-	flags.StringVar(&TestContext.DockerConfigFile, "docker-config-file", "", "A file that contains credentials which can be used to pull images from certain private registries, needed for a test.")
+	flags.StringVar(&TestContext.DockerConfigFile, "docker-config-file", "", "A docker credential file which contains authorization token that is used to perform image pull tests from an authenticated registry. For more details regarding the content of the file refer https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#log-in-to-docker-hub")
+
+	flags.StringVar(&TestContext.E2EDockerConfigFile, "e2e-docker-config-file", "", "A docker credentials configuration file used which contains authorization token that can be used to pull images from certain private registries provided by the users. For more details refer https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#log-in-to-docker-hub")
+	flags.StringVar(&TestContext.KubeTestRepoList, "kube-test-repo-list", "", "A yaml file used for overriding registries for test images. Alternatively, the KUBE_TEST_REPO_LIST env variable can be set.")
 
 	flags.StringVar(&TestContext.SnapshotControllerPodName, "snapshot-controller-pod-name", "", "The pod name to use for identifying the snapshot controller in the kube-system namespace.")
 	flags.IntVar(&TestContext.SnapshotControllerHTTPPort, "snapshot-controller-http-port", 0, "The port to use for snapshot controller HTTP communication.")
@@ -463,6 +473,9 @@ func AfterReadingAllFlags(t *TestContextType) {
 
 	// These flags are not exposed via the normal command line flag set,
 	// therefore we have to use our own private one here.
+	if t.KubeTestRepoList != "" {
+		image.Init(t.KubeTestRepoList)
+	}
 	var fs flag.FlagSet
 	klog.InitFlags(&fs)
 	fs.Set("logtostderr", "false")

test/e2e/framework/util.go

@@ -318,7 +318,7 @@ func waitForServiceAccountInNamespace(ctx context.Context, c clientset.Interface
 // the default service account is what is associated with pods when they do not specify a service account
 // as a result, pods are not able to be provisioned in a namespace until the service account is provisioned
 func WaitForDefaultServiceAccountInNamespace(ctx context.Context, c clientset.Interface, namespace string) error {
-	return waitForServiceAccountInNamespace(ctx, c, namespace, "default", ServiceAccountProvisionTimeout)
+	return waitForServiceAccountInNamespace(ctx, c, namespace, defaultServiceAccountName, ServiceAccountProvisionTimeout)
 }
 
 // WaitForKubeRootCAInNamespace waits for the configmap kube-root-ca.crt containing the service account

test/utils/image/manifest.go

@@ -68,12 +68,16 @@ func (i *Config) SetVersion(version string) {
 	i.version = version
 }
 
-func initReg() RegistryList {
+func Init(repoList string) {
+	registry, imageConfigs, originalImageConfigs = readRepoList(repoList)
+}
+
+func readRepoList(repoList string) (RegistryList, map[ImageID]Config, map[ImageID]Config) {
 	registry := initRegistry
 
-	repoList := os.Getenv("KUBE_TEST_REPO_LIST")
 	if repoList == "" {
-		return registry
+		imageConfigs, originalImageConfigs := initImageConfigs(registry)
+		return registry, imageConfigs, originalImageConfigs
 	}
 
 	var fileContent []byte
@@ -94,9 +98,13 @@ func initReg() RegistryList {
 
 	err = yaml.Unmarshal(fileContent, &registry)
 	if err != nil {
-		panic(fmt.Errorf("Error unmarshalling '%v' YAML file: %v", repoList, err))
+		panic(fmt.Errorf("error unmarshalling '%v' YAML file: %v", repoList, err))
 	}
-	return registry
+
+	imageConfigs, originalImageConfigs := initImageConfigs(registry)
+
+	return registry, imageConfigs, originalImageConfigs
+
 }
 
 // Essentially curl url | writer
@@ -135,10 +143,7 @@ var (
 		CloudProviderGcpRegistry: "registry.k8s.io/cloud-provider-gcp",
 	}
 
-	registry = initReg()
+	registry, imageConfigs, originalImageConfigs = readRepoList(os.Getenv("KUBE_TEST_REPO_LIST"))
-
-	// Preconfigured image configs
-	imageConfigs, originalImageConfigs = initImageConfigs(registry)
 )
 
 type ImageID int