Merge pull request #74703 from johscheuer/refactor-csi-e2e-tests

Add patch method for container images

test/e2e/framework/create.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pkg/errors"
 
 	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -34,6 +34,7 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/cache"
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
@@ -355,8 +356,20 @@ func (f *Framework) patchItemRecursively(item interface{}) error {
 		f.PatchNamespace(&item.ObjectMeta.Namespace)
 	case *appsv1.StatefulSet:
 		f.PatchNamespace(&item.ObjectMeta.Namespace)
+		if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.Containers); err != nil {
+			return err
+		}
+		if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil {
+			return err
+		}
 	case *appsv1.DaemonSet:
 		f.PatchNamespace(&item.ObjectMeta.Namespace)
+		if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.Containers); err != nil {
+			return err
+		}
+		if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil {
+			return err
+		}
 	default:
 		return errors.Errorf("missing support for patching item of type %T", item)
 	}

test/e2e/framework/pod/resource.go

@@ -688,3 +688,17 @@ func GetPodsScheduled(masterNodes sets.String, pods *v1.PodList) (scheduledPods,
 	}
 	return
 }
+
+// PatchContainerImages replaces the specified Container Registry with a custom
+// one provided via the KUBE_TEST_REPO_LIST env variable
+func PatchContainerImages(containers []v1.Container) error {
+	var err error
+	for _, c := range containers {
+		c.Image, err = imageutils.ReplaceRegistryInImageURL(c.Image)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

test/utils/image/BUILD

@@ -1,9 +1,6 @@
 package(default_visibility = ["//visibility:public"])
 
-load(
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
 
 go_library(
     name = "go_default_library",
@@ -26,3 +23,9 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["manifest_test.go"],
+    embed = [":go_default_library"],
+)

test/utils/image/manifest.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 
 	yaml "gopkg.in/yaml.v2"
 )
@@ -31,9 +32,11 @@ type RegistryList struct {
 	E2eRegistry             string `yaml:"e2eRegistry"`
 	InvalidRegistry         string `yaml:"invalidRegistry"`
 	GcRegistry              string `yaml:"gcRegistry"`
+	GcrReleaseRegistry      string `yaml:"gcrReleaseRegistry"`
 	GoogleContainerRegistry string `yaml:"googleContainerRegistry"`
 	PrivateRegistry         string `yaml:"privateRegistry"`
 	SampleRegistry          string `yaml:"sampleRegistry"`
+	QuayK8sCSI              string `yaml:"quayK8sCSI"`
 }
 
 // Config holds an images registry, name, and version
@@ -65,9 +68,11 @@ func initReg() RegistryList {
 		E2eRegistry:             "gcr.io/kubernetes-e2e-test-images",
 		InvalidRegistry:         "invalid.com/invalid",
 		GcRegistry:              "k8s.gcr.io",
+		GcrReleaseRegistry:      "gcr.io/gke-release",
 		GoogleContainerRegistry: "gcr.io/google-containers",
 		PrivateRegistry:         "gcr.io/k8s-authenticated-test",
 		SampleRegistry:          "gcr.io/google-samples",
+		QuayK8sCSI:              "quay.io/k8scsi",
 	}
 	repoList := os.Getenv("KUBE_TEST_REPO_LIST")
 	if repoList == "" {
@@ -93,8 +98,10 @@ var (
 	e2eGcRegistry           = "gcr.io/kubernetes-e2e-test-images"
 	gcAuthenticatedRegistry = registry.GcAuthenticatedRegistry
 	gcRegistry              = registry.GcRegistry
+	gcrReleaseRegistry      = registry.GcrReleaseRegistry
 	googleContainerRegistry = registry.GoogleContainerRegistry
 	invalidRegistry         = registry.InvalidRegistry
+	quayK8sCSI              = registry.QuayK8sCSI
 	// PrivateRegistry is an image repository that requires authentication
 	PrivateRegistry = registry.PrivateRegistry
 	sampleRegistry  = registry.SampleRegistry
@@ -269,3 +276,38 @@ func (i *Config) GetE2EImage() string {
 func GetPauseImageName() string {
 	return GetE2EImage(Pause)
 }
+
+// ReplaceRegistryInImageURL replaces the registry in the image URL with a custom one
+func ReplaceRegistryInImageURL(imageURL string) (string, error) {
+	parts := strings.Split(imageURL, "/")
+	countParts := len(parts)
+	registryAndUser := strings.Join(parts[:countParts-1], "/")
+
+	switch registryAndUser {
+	case "gcr.io/kubernetes-e2e-test-images":
+		registryAndUser = e2eRegistry
+	case "k8s.gcr.io":
+		registryAndUser = gcRegistry
+	case "gcr.io/k8s-authenticated-test":
+		registryAndUser = PrivateRegistry
+	case "gcr.io/google-samples":
+		registryAndUser = sampleRegistry
+	case "gcr.io/gke-release":
+		registryAndUser = gcrReleaseRegistry
+	case "docker.io/library":
+		registryAndUser = dockerLibraryRegistry
+	case "quay.io/k8scsi":
+		registryAndUser = quayK8sCSI
+	default:
+		if countParts == 1 {
+			// We assume we found an image from docker hub library
+			// e.g. openjdk -> docker.io/library/openjdk
+			registryAndUser = dockerLibraryRegistry
+			break
+		}
+
+		return "", fmt.Errorf("Registry: %s is missing in test/utils/image/manifest.go, please add the registry, otherwise the test will fail on air-gapped clusters", registryAndUser)
+	}
+
+	return fmt.Sprintf("%s/%s", registryAndUser, parts[countParts-1]), nil
+}

test/utils/image/manifest_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package image
+
+import (
+	"fmt"
+	"testing"
+)
+
+type result struct {
+	result string
+	err    error
+}
+
+var registryTests = []struct {
+	in  string
+	out result
+}{
+	{
+		"docker.io/library/test:123",
+		result{
+			result: "test.io/library/test:123",
+			err:    nil,
+		},
+	},
+	{
+		"docker.io/library/test",
+		result{
+			result: "test.io/library/test",
+			err:    nil,
+		},
+	},
+	{
+		"test",
+		result{
+			result: "test.io/library/test",
+			err:    nil,
+		},
+	},
+	{
+		"gcr.io/kubernetes-e2e-test-images/test:123",
+		result{
+			result: "test.io/kubernetes-e2e-test-images/test:123",
+			err:    nil,
+		},
+	},
+	{
+		"k8s.gcr.io/test:123",
+		result{
+			result: "test.io/test:123",
+			err:    nil,
+		},
+	},
+	{
+		"gcr.io/k8s-authenticated-test/test:123",
+		result{
+			result: "test.io/k8s-authenticated-test/test:123",
+			err:    nil,
+		},
+	},
+	{
+		"gcr.io/gke-release/test:latest",
+		result{
+			result: "test.io/gke-release/test:latest",
+			err:    nil,
+		},
+	},
+	{
+		"gcr.io/google-samples/test:latest",
+		result{
+			result: "test.io/google-samples/test:latest",
+			err:    nil,
+		},
+	},
+	{
+		"quay.io/k8scsi/test:latest",
+		result{
+			result: "test.io/k8scsi/test:latest",
+			err:    nil,
+		},
+	},
+	{
+		"unknwon.io/google-samples/test:latest",
+		result{
+			result: "",
+			err:    fmt.Errorf("Registry: unknwon.io/google-samples is missing in test/utils/image/manifest.go, please add the registry, otherwise the test will fail on air-gapped clusters"),
+		},
+	},
+}
+
+// ToDo Add Benchmark
+func TestReplaceRegistryInImageURL(t *testing.T) {
+	// Set custom registries
+	dockerLibraryRegistry = "test.io/library"
+	e2eRegistry = "test.io/kubernetes-e2e-test-images"
+	gcRegistry = "test.io"
+	gcrReleaseRegistry = "test.io/gke-release"
+	PrivateRegistry = "test.io/k8s-authenticated-test"
+	sampleRegistry = "test.io/google-samples"
+	quayK8sCSI = "test.io/k8scsi"
+
+	for _, tt := range registryTests {
+		t.Run(tt.in, func(t *testing.T) {
+			s, err := ReplaceRegistryInImageURL(tt.in)
+
+			if err != nil && err.Error() != tt.out.err.Error() {
+				t.Errorf("got %q, want %q", err, tt.out.err)
+			}
+
+			if s != tt.out.result {
+				t.Errorf("got %q, want %q", s, tt.out.result)
+			}
+		})
+	}
+}