From df1d1cc26382777de65c608f287b82bf54c4d1d0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 2 Mar 2022 17:53:50 +0100 Subject: [PATCH] e2e: support CSI images in -list-images It was possible to patch images in the YAML files via KUBE_TEST_REPO, but it was not possible to know in advance which images might be needed. Now -list-images also includes all images referenced by the test-manifest/storage-csi YAML files. --- test/utils/image/csi_manifest.go | 133 +++++++++++++++++++++++++ test/utils/image/csi_manifests_test.go | 62 ++++++++++++ test/utils/image/manifest.go | 5 + 3 files changed, 200 insertions(+) create mode 100644 test/utils/image/csi_manifest.go create mode 100644 test/utils/image/csi_manifests_test.go diff --git a/test/utils/image/csi_manifest.go b/test/utils/image/csi_manifest.go new file mode 100644 index 00000000000..ca8f9c45322 --- /dev/null +++ b/test/utils/image/csi_manifest.go @@ -0,0 +1,133 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "bytes" + "fmt" + "io/fs" + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/test/e2e/testing-manifests" +) + +// All of the image tags are of the format k8s.gcr.io/sig-storage/hostpathplugin:v1.7.3. +var imageRE = regexp.MustCompile(`^(.*)/([^/:]*):(.*)$`) + +// appendCSIImageConfigs extracts image repo, name and version from +// the YAML files under test/e2e/testing-manifests/storage-csi and +// creates new config entries for them. +func appendCSIImageConfigs(configs map[int]Config) { + embeddedFS := testing_manifests.GetE2ETestingManifestsFS().EmbeddedFS + + // We add our images with index numbers that start after the highest existing number. + index := 0 + for i := range configs { + if i > index { + index = i + } + } + + err := fs.WalkDir(embeddedFS, "storage-csi", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + data, err := embeddedFS.ReadFile(path) + if err != nil { + return err + } + + // Split at the "---" separator before working on + // individual item. Only works for .yaml. + // + // We need to split ourselves because we need access + // to each original chunk of data for + // runtime.DecodeInto. kubectl has its own + // infrastructure for this, but that is a lot of code + // with many dependencies. + items := bytes.Split(data, []byte("\n---")) + for i, item := range items { + // We don't care what the actual type is. We just + // unmarshal into generic maps and then look up images. + var object interface{} + if err := yaml.Unmarshal(item, &object); err != nil { + return fmt.Errorf("decode item #%d in %s: %v", + i, path, err) + } + + // Will be called for all image strings. + visit := func(value string) { + parts := imageRE.FindStringSubmatch(value) + if parts == nil { + return + } + config := Config{parts[1], parts[2], parts[3]} + for _, otherConfig := range configs { + if otherConfig == config { + return + } + } + index++ + configs[index] = config + } + + // These paths match plain Pods and more complex types + // like Deployments. + findStrings(object, visit, "spec", "containers", "image") + findStrings(object, visit, "spec", "template", "spec", "containers", "image") + + } + return nil + + }) + if err != nil { + panic(err) + } +} + +// findStrings recursively decends into an object along a certain path. Path +// elements are the named fields. If a field references a list, each of the +// list elements will be followed. +// +// Conceptually this is similar to a JSON path. +func findStrings(object interface{}, visit func(value string), path ...string) { + if len(path) == 0 { + // Found it. May or may not be a string, though. + if object, ok := object.(string); ok { + visit(object) + } + return + } + + switch object := object.(type) { + case []interface{}: + // If we are in a list, check each entry. + for _, child := range object { + findStrings(child, visit, path...) + } + case map[string]interface{}: + // Follow path if possible + if child, ok := object[path[0]]; ok { + findStrings(child, visit, path[1:]...) + } + } +} diff --git a/test/utils/image/csi_manifests_test.go b/test/utils/image/csi_manifests_test.go new file mode 100644 index 00000000000..d522c0c04b4 --- /dev/null +++ b/test/utils/image/csi_manifests_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestCSIImageConfigs(t *testing.T) { + configs := map[int]Config{} + appendCSIImageConfigs(configs) + + // We expect at least one entry for each of these images. There may be + // more than one entry for the same image when different YAMLs use + // different versions. + // + // The exact versions are not checked here because that would bring + // back the problem of updating the expected versions. The set of + // images shouldn't change much. + expectedImages := []string{ + "csi-attacher", + "csi-external-health-monitor-controller", + "csi-node-driver-registrar", + "csi-provisioner", + "csi-resizer", + "csi-snapshotter", + "hostpathplugin", + "livenessprobe", + + // From the GCP deployment. + "gcp-compute-persistent-disk-csi-driver", + + // For some hostpath tests. + "socat", + "busybox", + } + actualImages := sets.NewString() + for _, config := range configs { + assert.NotEmpty(t, config.registry, "registry") + assert.NotEmpty(t, config.name, "name") + assert.NotEmpty(t, config.version, "version") + actualImages.Insert(config.name) + } + assert.ElementsMatch(t, expectedImages, actualImages.UnsortedList(), "found these images: %+v", configs) +} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 0715e2d9868..f950c5bec43 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -241,6 +241,11 @@ func initImageConfigs(list RegistryList) (map[int]Config, map[int]Config) { configs[VolumeRBDServer] = Config{list.PromoterE2eRegistry, "volume/rbd", "1.0.4"} configs[WindowsServer] = Config{list.MicrosoftRegistry, "windows", "1809"} + // This adds more config entries. Those have no pre-defined index number, + // but will be used via ReplaceRegistryInImageURL when deploying + // CSI drivers (test/e2e/storage/util/create.go). + appendCSIImageConfigs(configs) + // if requested, map all the SHAs into a known format based on the input originalImageConfigs := configs if repo := os.Getenv("KUBE_TEST_REPO"); len(repo) > 0 {