diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index f877413f0ea..773e980a44d 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -4,7 +4,6 @@ go_library( name = "go_default_library", srcs = [ "cleanup.go", - "create.go", "exec_util.go", "expect.go", "flake_reporting_util.go", @@ -44,8 +43,6 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", @@ -71,7 +68,6 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/restmapper:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", @@ -86,7 +82,6 @@ go_library( "//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/resource:go_default_library", "//test/e2e/framework/ssh:go_default_library", - "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/manifest:go_default_library", "//test/e2e/system:go_default_library", "//test/utils:go_default_library", diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 8959bdc0e97..03b2806f076 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -187,7 +187,7 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per SnapshotterContainerName: "csi-snapshotter", NodeName: node.Name, } - cleanup, err := config.Framework.CreateFromManifests(func(item interface{}) error { + cleanup, err := utils.CreateFromManifests(config.Framework, func(item interface{}) error { return utils.PatchCSIDeployment(config.Framework, o, item) }, h.manifests...) @@ -329,7 +329,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest storagev1beta1.VolumeLifecycleEphemeral, }, } - cleanup, err := f.CreateFromManifests(func(item interface{}) error { + cleanup, err := utils.CreateFromManifests(f, func(item interface{}) error { return utils.PatchCSIDeployment(f, o, item) }, m.manifests...) @@ -459,7 +459,7 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", } - cleanup, err := f.CreateFromManifests(nil, manifests...) + cleanup, err := utils.CreateFromManifests(f, nil, manifests...) if err != nil { framework.Failf("deploying csi gce-pd driver: %v", err) } diff --git a/test/e2e/storage/external/BUILD b/test/e2e/storage/external/BUILD index dbadb6899fe..5a81576e933 100644 --- a/test/e2e/storage/external/BUILD +++ b/test/e2e/storage/external/BUILD @@ -18,6 +18,7 @@ go_library( "//test/e2e/framework/volume:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/testsuites:go_default_library", + "//test/e2e/storage/utils:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index bb9957a3896..55ac5bf0c67 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" + "k8s.io/kubernetes/test/e2e/storage/utils" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -271,11 +272,11 @@ func (d *driverDefinition) GetDynamicProvisionStorageClass(config *testsuites.Pe return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } - items, err := f.LoadFromManifests(d.StorageClass.FromFile) + items, err := utils.LoadFromManifests(d.StorageClass.FromFile) framework.ExpectNoError(err, "load storage class from %s", d.StorageClass.FromFile) framework.ExpectEqual(len(items), 1, "exactly one item from %s", d.StorageClass.FromFile) - err = f.PatchItems(items...) + err = utils.PatchItems(f, items...) framework.ExpectNoError(err, "patch items") sc, ok := items[0].(*storagev1.StorageClass) diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD index 6c761e9a3df..83795177c4d 100644 --- a/test/e2e/storage/utils/BUILD +++ b/test/e2e/storage/utils/BUILD @@ -5,6 +5,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "create.go", "deployment.go", "framework.go", "host_exec.go", @@ -20,19 +21,25 @@ go_library( "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/exec:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/ssh:go_default_library", + "//test/e2e/framework/testfiles:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/test/e2e/framework/create.go b/test/e2e/storage/utils/create.go similarity index 84% rename from test/e2e/framework/create.go rename to test/e2e/storage/utils/create.go index 04f0c58c36f..d1b7ab6f26a 100644 --- a/test/e2e/framework/create.go +++ b/test/e2e/storage/utils/create.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework +package utils import ( "bytes" @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" + "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/framework/testfiles" ) @@ -51,7 +52,7 @@ import ( // and silently ignored // - the latest stable API version for each item is used, regardless of what // is specified in the manifest files -func (f *Framework) LoadFromManifests(files ...string) ([]interface{}, error) { +func LoadFromManifests(files ...string) ([]interface{}, error) { var items []interface{} err := visitManifests(func(data []byte) error { // Ignore any additional fields for now, just determine what we have. @@ -80,7 +81,7 @@ func visitManifests(cb func([]byte) error, files ...string) error { for _, fileName := range files { data, err := testfiles.Read(fileName) if err != nil { - Failf("reading manifest file: %v", err) + framework.Failf("reading manifest file: %v", err) } // Split at the "---" separator before working on @@ -113,11 +114,11 @@ func visitManifests(cb func([]byte) error, files ...string) error { // PatchItems has some limitations: // - only some common items are supported, unknown ones trigger an error // - only the latest stable API version for each item is supported -func (f *Framework) PatchItems(items ...interface{}) error { +func PatchItems(f *framework.Framework, items ...interface{}) error { for _, item := range items { // Uncomment when debugging the loading and patching of items. // Logf("patching original content of %T:\n%s", item, PrettyPrint(item)) - if err := f.patchItemRecursively(item); err != nil { + if err := patchItemRecursively(f, item); err != nil { return err } } @@ -138,15 +139,15 @@ func (f *Framework) PatchItems(items ...interface{}) error { // PatchItems has the some limitations as LoadFromManifests: // - only some common items are supported, unknown ones trigger an error // - only the latest stable API version for each item is supported -func (f *Framework) CreateItems(items ...interface{}) (func(), error) { +func CreateItems(f *framework.Framework, items ...interface{}) (func(), error) { var destructors []func() error - var cleanupHandle CleanupActionHandle + var cleanupHandle framework.CleanupActionHandle cleanup := func() { if cleanupHandle == nil { // Already done. return } - RemoveCleanupAction(cleanupHandle) + framework.RemoveCleanupAction(cleanupHandle) // TODO (?): use same logic as framework.go for determining // whether we are expected to clean up? This would change the @@ -155,11 +156,11 @@ func (f *Framework) CreateItems(items ...interface{}) (func(), error) { // to non-namespaced items. for _, destructor := range destructors { if err := destructor(); err != nil && !apierrs.IsNotFound(err) { - Logf("deleting failed: %s", err) + framework.Logf("deleting failed: %s", err) } } } - cleanupHandle = AddCleanupAction(cleanup) + cleanupHandle = framework.AddCleanupAction(cleanup) var result error for _, item := range items { @@ -168,12 +169,12 @@ func (f *Framework) CreateItems(items ...interface{}) (func(), error) { description := describeItem(item) // Uncomment this line to get a full dump of the entire item. // description = fmt.Sprintf("%s:\n%s", description, PrettyPrint(item)) - Logf("creating %s", description) + framework.Logf("creating %s", description) for _, factory := range factories { destructor, err := factory.Create(f, item) if destructor != nil { destructors = append(destructors, func() error { - Logf("deleting %s", description) + framework.Logf("deleting %s", description) return destructor() }) } @@ -202,12 +203,12 @@ func (f *Framework) CreateItems(items ...interface{}) (func(), error) { // CreateFromManifests is a combination of LoadFromManifests, // PatchItems, patching with an optional custom function, // and CreateItems. -func (f *Framework) CreateFromManifests(patch func(item interface{}) error, files ...string) (func(), error) { - items, err := f.LoadFromManifests(files...) +func CreateFromManifests(f *framework.Framework, patch func(item interface{}) error, files ...string) (func(), error) { + items, err := LoadFromManifests(files...) if err != nil { return nil, errors.Wrap(err, "CreateFromManifests") } - if err := f.PatchItems(items...); err != nil { + if err := PatchItems(f, items...); err != nil { return nil, err } if patch != nil { @@ -217,7 +218,7 @@ func (f *Framework) CreateFromManifests(patch func(item interface{}) error, file } } } - return f.CreateItems(items...) + return CreateItems(f, items...) } // What is a subset of metav1.TypeMeta which (in contrast to @@ -257,7 +258,7 @@ type ItemFactory interface { // error or a cleanup function for the created item. // If the item is of an unsupported type, it must return // an error that has errorItemNotSupported as cause. - Create(f *Framework, item interface{}) (func() error, error) + Create(f *framework.Framework, item interface{}) (func() error, error) } // describeItem always returns a string that describes the item, @@ -292,7 +293,7 @@ var factories = map[What]ItemFactory{ // PatchName makes the name of some item unique by appending the // generated unique name. -func (f *Framework) PatchName(item *string) { +func PatchName(f *framework.Framework, item *string) { if *item != "" { *item = *item + "-" + f.UniqueName } @@ -301,64 +302,64 @@ func (f *Framework) PatchName(item *string) { // PatchNamespace moves the item into the test's namespace. Not // all items can be namespaced. For those, the name also needs to be // patched. -func (f *Framework) PatchNamespace(item *string) { +func PatchNamespace(f *framework.Framework, item *string) { if f.Namespace != nil { *item = f.Namespace.GetName() } } -func (f *Framework) patchItemRecursively(item interface{}) error { +func patchItemRecursively(f *framework.Framework, item interface{}) error { switch item := item.(type) { case *rbacv1.Subject: - f.PatchNamespace(&item.Namespace) + PatchNamespace(f, &item.Namespace) case *rbacv1.RoleRef: // TODO: avoid hard-coding this special name. Perhaps add a Framework.PredefinedRoles // which contains all role names that are defined cluster-wide before the test starts? // All those names are excempt from renaming. That list could be populated by querying // and get extended by tests. if item.Name != "e2e-test-privileged-psp" { - f.PatchName(&item.Name) + PatchName(f, &item.Name) } case *rbacv1.ClusterRole: - f.PatchName(&item.Name) + PatchName(f, &item.Name) case *rbacv1.Role: - f.PatchNamespace(&item.Namespace) + PatchNamespace(f, &item.Namespace) // Roles are namespaced, but because for RoleRef above we don't // know whether the referenced role is a ClusterRole or Role // and therefore always renames, we have to do the same here. - f.PatchName(&item.Name) + PatchName(f, &item.Name) case *storagev1.StorageClass: - f.PatchName(&item.Name) + PatchName(f, &item.Name) case *storagev1beta1.CSIDriver: - f.PatchName(&item.Name) + PatchName(f, &item.Name) case *v1.ServiceAccount: - f.PatchNamespace(&item.ObjectMeta.Namespace) + PatchNamespace(f, &item.ObjectMeta.Namespace) case *v1.Secret: - f.PatchNamespace(&item.ObjectMeta.Namespace) + PatchNamespace(f, &item.ObjectMeta.Namespace) case *rbacv1.ClusterRoleBinding: - f.PatchName(&item.Name) + PatchName(f, &item.Name) for i := range item.Subjects { - if err := f.patchItemRecursively(&item.Subjects[i]); err != nil { + if err := patchItemRecursively(f, &item.Subjects[i]); err != nil { return errors.Wrapf(err, "%T", f) } } - if err := f.patchItemRecursively(&item.RoleRef); err != nil { + if err := patchItemRecursively(f, &item.RoleRef); err != nil { return errors.Wrapf(err, "%T", f) } case *rbacv1.RoleBinding: - f.PatchNamespace(&item.Namespace) + PatchNamespace(f, &item.Namespace) for i := range item.Subjects { - if err := f.patchItemRecursively(&item.Subjects[i]); err != nil { + if err := patchItemRecursively(f, &item.Subjects[i]); err != nil { return errors.Wrapf(err, "%T", f) } } - if err := f.patchItemRecursively(&item.RoleRef); err != nil { + if err := patchItemRecursively(f, &item.RoleRef); err != nil { return errors.Wrapf(err, "%T", f) } case *v1.Service: - f.PatchNamespace(&item.ObjectMeta.Namespace) + PatchNamespace(f, &item.ObjectMeta.Namespace) case *appsv1.StatefulSet: - f.PatchNamespace(&item.ObjectMeta.Namespace) + PatchNamespace(f, &item.ObjectMeta.Namespace) if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -366,7 +367,7 @@ func (f *Framework) patchItemRecursively(item interface{}) error { return err } case *appsv1.DaemonSet: - f.PatchNamespace(&item.ObjectMeta.Namespace) + PatchNamespace(f, &item.ObjectMeta.Namespace) if err := e2epod.PatchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -390,7 +391,7 @@ func (f *serviceAccountFactory) New() runtime.Object { return &v1.ServiceAccount{} } -func (*serviceAccountFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*serviceAccountFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*v1.ServiceAccount) if !ok { return nil, errorItemNotSupported @@ -410,13 +411,13 @@ func (f *clusterRoleFactory) New() runtime.Object { return &rbacv1.ClusterRole{} } -func (*clusterRoleFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*clusterRoleFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*rbacv1.ClusterRole) if !ok { return nil, errorItemNotSupported } - Logf("Define cluster role %v", item.GetName()) + framework.Logf("Define cluster role %v", item.GetName()) client := f.ClientSet.RbacV1().ClusterRoles() if _, err := client.Create(item); err != nil { return nil, errors.Wrap(err, "create ClusterRole") @@ -432,7 +433,7 @@ func (f *clusterRoleBindingFactory) New() runtime.Object { return &rbacv1.ClusterRoleBinding{} } -func (*clusterRoleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*clusterRoleBindingFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*rbacv1.ClusterRoleBinding) if !ok { return nil, errorItemNotSupported @@ -453,7 +454,7 @@ func (f *roleFactory) New() runtime.Object { return &rbacv1.Role{} } -func (*roleFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*roleFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*rbacv1.Role) if !ok { return nil, errorItemNotSupported @@ -474,7 +475,7 @@ func (f *roleBindingFactory) New() runtime.Object { return &rbacv1.RoleBinding{} } -func (*roleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*roleBindingFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*rbacv1.RoleBinding) if !ok { return nil, errorItemNotSupported @@ -495,7 +496,7 @@ func (f *serviceFactory) New() runtime.Object { return &v1.Service{} } -func (*serviceFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*serviceFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*v1.Service) if !ok { return nil, errorItemNotSupported @@ -516,7 +517,7 @@ func (f *statefulSetFactory) New() runtime.Object { return &appsv1.StatefulSet{} } -func (*statefulSetFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*statefulSetFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*appsv1.StatefulSet) if !ok { return nil, errorItemNotSupported @@ -537,7 +538,7 @@ func (f *daemonSetFactory) New() runtime.Object { return &appsv1.DaemonSet{} } -func (*daemonSetFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*daemonSetFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*appsv1.DaemonSet) if !ok { return nil, errorItemNotSupported @@ -558,7 +559,7 @@ func (f *storageClassFactory) New() runtime.Object { return &storagev1.StorageClass{} } -func (*storageClassFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*storageClassFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*storagev1.StorageClass) if !ok { return nil, errorItemNotSupported @@ -579,7 +580,7 @@ func (f *csiDriverFactory) New() runtime.Object { return &storagev1beta1.CSIDriver{} } -func (*csiDriverFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*csiDriverFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*storagev1beta1.CSIDriver) if !ok { return nil, errorItemNotSupported @@ -600,7 +601,7 @@ func (f *secretFactory) New() runtime.Object { return &v1.Secret{} } -func (*secretFactory) Create(f *Framework, i interface{}) (func() error, error) { +func (*secretFactory) Create(f *framework.Framework, i interface{}) (func() error, error) { item, ok := i.(*v1.Secret) if !ok { return nil, errorItemNotSupported