From 386f94ff032813ce15426b9ff9e7250281af9d8f Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Tue, 28 Jul 2020 14:30:16 -0400 Subject: [PATCH] test/utils/image: Support a single repository In downstream contexts, it's extremely useful to be able to combine all the "testable" images in Kubernetes into a single repo so that a user could mirror these offline in one chunk, and audit the set of images for changes. For instance, within OpenShift we would like to have a single place we can place all the images used by all the tests with a single authentication scheme. While some images are not "real" and can't be mirrored (for instance, the images that point to an auth protected registry), that is not the majority. This code makes it possible to specify an environment variable KUBE_TEST_REPO that maps the static strings of the registry to a single repository by placing the uniqueness in a tag. For instance: KUBE_TEST_REPO=quay.io/openshift/community-e2e-images would translate `k8s.gcr.io/prometheus-to-sd:v0.5.0` to `quay.io/openshift/community-e2e-images:e2e-30-k8s-gcr-io-prometheus-to-sd-v0-5-0-6JI59Yih4oaj3oQOjRfhyQ`. The tag is a safe form of the name, plus the index (the constant within manifest.go), plus a hash of the full input. The length of the tag is constrained to the minimum of hash + index + the safe name. The public method is changed to return two maps - index to original name and index to test repo name. These maps would be the same if the env var is not set. --- test/utils/image/BUILD | 1 + test/utils/image/manifest.go | 113 +++++++++++++++++++++++++++--- test/utils/image/manifest_test.go | 28 ++++++++ 3 files changed, 133 insertions(+), 9 deletions(-) diff --git a/test/utils/image/BUILD b/test/utils/image/BUILD index f07ee05fc19..287c4c4b21e 100644 --- a/test/utils/image/BUILD +++ b/test/utils/image/BUILD @@ -28,4 +28,5 @@ go_test( name = "go_default_test", srcs = ["manifest_test.go"], embed = [":go_default_library"], + deps = ["//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library"], ) diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 2b9026c9ed7..a860fbf4c69 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -17,9 +17,12 @@ limitations under the License. package image import ( + "crypto/sha256" + "encoding/base64" "fmt" "io/ioutil" "os" + "regexp" "strings" yaml "gopkg.in/yaml.v2" @@ -100,7 +103,12 @@ func initReg() RegistryList { } var ( - registry = initReg() + registry = initReg() + + // PrivateRegistry is an image repository that requires authentication + PrivateRegistry = registry.PrivateRegistry + + // Preconfigured image configs dockerLibraryRegistry = registry.DockerLibraryRegistry dockerGluster = registry.DockerGluster e2eRegistry = registry.E2eRegistry @@ -112,14 +120,10 @@ var ( sigStorageRegistry = registry.SigStorageRegistry gcrReleaseRegistry = registry.GcrReleaseRegistry invalidRegistry = registry.InvalidRegistry - // PrivateRegistry is an image repository that requires authentication - PrivateRegistry = registry.PrivateRegistry - sampleRegistry = registry.SampleRegistry + sampleRegistry = registry.SampleRegistry + microsoftRegistry = registry.MicrosoftRegistry - // Preconfigured image configs - imageConfigs = initImageConfigs() - - microsoftRegistry = registry.MicrosoftRegistry + imageConfigs, originalImageConfigs = initImageConfigs() ) const ( @@ -206,7 +210,7 @@ const ( WindowsServer ) -func initImageConfigs() map[int]Config { +func initImageConfigs() (map[int]Config, map[int]Config) { configs := map[int]Config{} configs[Agnhost] = Config{promoterE2eRegistry, "agnhost", "2.21"} configs[AgnhostPrivate] = Config{PrivateRegistry, "agnhost", "2.6"} @@ -248,9 +252,83 @@ func initImageConfigs() map[int]Config { configs[VolumeGlusterServer] = Config{e2eVolumeRegistry, "gluster", "1.0"} configs[VolumeRBDServer] = Config{e2eVolumeRegistry, "rbd", "1.0.1"} configs[WindowsServer] = Config{microsoftRegistry, "windows", "1809"} + + // if requested, map all the SHAs into a known format based on the input + originalImageConfigs := configs + if repo := os.Getenv("KUBE_TEST_REPO"); len(repo) > 0 { + configs = GetMappedImageConfigs(originalImageConfigs, repo) + } + + return configs, originalImageConfigs +} + +// GetMappedImageConfigs returns the images if they were mapped to the provided +// image repository. +func GetMappedImageConfigs(originalImageConfigs map[int]Config, repo string) map[int]Config { + configs := make(map[int]Config) + for i, config := range originalImageConfigs { + switch i { + case InvalidRegistryImage, AuthenticatedAlpine, + AuthenticatedWindowsNanoServer, AgnhostPrivate: + // These images are special and can't be run out of the cloud - some because they + // are authenticated, and others because they are not real images. Tests that depend + // on these images can't be run without access to the public internet. + configs[i] = config + continue + } + + // Build a new tag with a the index, a hash of the image spec (to be unique) and + // shorten and make the pull spec "safe" so it will fit in the tag + configs[i] = getRepositoryMappedConfig(i, config, repo) + } return configs } +var ( + reCharSafe = regexp.MustCompile(`[^\w]`) + reDashes = regexp.MustCompile(`-+`) +) + +// getRepositoryMappedConfig maps an existing image to the provided repo, generating a +// tag that is unique with the input config. The tag will contain the index, a hash of +// the image spec (to be unique) and shorten and make the pull spec "safe" so it will +// fit in the tag to allow a human to recognize the value. If index is -1, then no +// index will be added to the tag. +func getRepositoryMappedConfig(index int, config Config, repo string) Config { + parts := strings.SplitN(repo, "/", 2) + registry, name := parts[0], parts[1] + + pullSpec := config.GetE2EImage() + + h := sha256.New() + h.Write([]byte(pullSpec)) + hash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)[:16]) + + shortName := reCharSafe.ReplaceAllLiteralString(pullSpec, "-") + shortName = reDashes.ReplaceAllLiteralString(shortName, "-") + maxLength := 127 - 16 - 6 - 10 + if len(shortName) > maxLength { + shortName = shortName[len(shortName)-maxLength:] + } + var version string + if index == -1 { + version = fmt.Sprintf("e2e-%s-%s", shortName, hash) + } else { + version = fmt.Sprintf("e2e-%d-%s-%s", index, shortName, hash) + } + + return Config{ + registry: registry, + name: name, + version: version, + } +} + +// GetOriginalImageConfigs returns the configuration before any mapping rules. +func GetOriginalImageConfigs() map[int]Config { + return originalImageConfigs +} + // GetImageConfigs returns the map of imageConfigs func GetImageConfigs() map[int]Config { return imageConfigs @@ -282,6 +360,23 @@ func ReplaceRegistryInImageURL(imageURL string) (string, error) { countParts := len(parts) registryAndUser := strings.Join(parts[:countParts-1], "/") + if repo := os.Getenv("KUBE_TEST_REPO"); len(repo) > 0 { + index := -1 + for i, v := range originalImageConfigs { + if v.GetE2EImage() == imageURL { + index = i + break + } + } + last := strings.SplitN(parts[countParts-1], ":", 2) + config := getRepositoryMappedConfig(index, Config{ + registry: parts[0], + name: strings.Join([]string{strings.Join(parts[1:countParts-1], "/"), last[0]}, "/"), + version: last[1], + }, repo) + return config.GetE2EImage(), nil + } + switch registryAndUser { case "gcr.io/kubernetes-e2e-test-images": registryAndUser = e2eRegistry diff --git a/test/utils/image/manifest_test.go b/test/utils/image/manifest_test.go index 938af362f08..7f7e444de1b 100644 --- a/test/utils/image/manifest_test.go +++ b/test/utils/image/manifest_test.go @@ -18,7 +18,10 @@ package image import ( "fmt" + "reflect" "testing" + + "k8s.io/apimachinery/pkg/util/diff" ) type result struct { @@ -135,3 +138,28 @@ func TestReplaceRegistryInImageURL(t *testing.T) { }) } } + +func TestGetOriginalImageConfigs(t *testing.T) { + if len(GetOriginalImageConfigs()) == 0 { + t.Fatalf("original map should not be empty") + } +} + +func TestGetMappedImageConfigs(t *testing.T) { + originals := map[int]Config{ + 0: {registry: "docker.io", name: "source/repo", version: "1.0"}, + } + mapping := GetMappedImageConfigs(originals, "quay.io/repo/for-test") + + actual := make(map[string]string) + for i, mapping := range mapping { + source := originals[i] + actual[source.GetE2EImage()] = mapping.GetE2EImage() + } + expected := map[string]string{ + "docker.io/source/repo:1.0": "quay.io/repo/for-test:e2e-0-docker-io-source-repo-1-0-72R4aXm7YnxQ4_ekf1DrFA", + } + if !reflect.DeepEqual(expected, actual) { + t.Fatal(diff.ObjectReflectDiff(expected, actual)) + } +}