diff --git a/test/e2e/auth/BUILD b/test/e2e/auth/BUILD index ee57ae784eb..515868770a0 100644 --- a/test/e2e/auth/BUILD +++ b/test/e2e/auth/BUILD @@ -51,7 +51,6 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/auth:go_default_library", "//test/e2e/framework/deployment:go_default_library", diff --git a/test/e2e/auth/pod_security_policy.go b/test/e2e/auth/pod_security_policy.go index ad14029e592..1c6a46f76cf 100644 --- a/test/e2e/auth/pod_security_policy.go +++ b/test/e2e/auth/pod_security_policy.go @@ -31,7 +31,6 @@ import ( "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/pkg/security/podsecuritypolicy/seccomp" psputil "k8s.io/kubernetes/pkg/security/podsecuritypolicy/util" - "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/auth" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -168,7 +167,7 @@ func testPrivilegedPods(tester func(pod *v1.Pod)) { tester(hostipc) }) - if common.IsAppArmorSupported() { + if framework.IsAppArmorSupported() { ginkgo.By("Running a custom AppArmor profile pod", func() { aa := restrictedPod("apparmor") // Every node is expected to have the docker-default profile. diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index 30033f0702a..9e69f3de340 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -8,7 +8,6 @@ load( go_library( name = "go_default_library", srcs = [ - "apparmor.go", "configmap.go", "configmap_volume.go", "container.go", @@ -51,7 +50,6 @@ go_library( "//pkg/kubelet/images:go_default_library", "//pkg/kubelet/runtimeclass/testing:go_default_library", "//pkg/kubelet/sysctl:go_default_library", - "//pkg/security/apparmor:go_default_library", "//staging/src/k8s.io/api/coordination/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 767dfabd131..6c90d70c5a7 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "checks.go", "cleanup.go", "exec_util.go", "expect.go", @@ -134,6 +135,7 @@ filegroup( "//test/e2e/framework/pv:all-srcs", "//test/e2e/framework/replicaset:all-srcs", "//test/e2e/framework/resource:all-srcs", + "//test/e2e/framework/security:all-srcs", "//test/e2e/framework/service:all-srcs", "//test/e2e/framework/ssh:all-srcs", "//test/e2e/framework/statefulset:all-srcs", diff --git a/test/e2e/framework/checks.go b/test/e2e/framework/checks.go new file mode 100644 index 00000000000..511ec48e5b9 --- /dev/null +++ b/test/e2e/framework/checks.go @@ -0,0 +1,22 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +// IsAppArmorSupported checks whether the AppArmor is supported by the node OS distro. +func IsAppArmorSupported() bool { + return NodeOSDistroIs(AppArmorDistros...) +} diff --git a/test/e2e/framework/security/BUILD b/test/e2e/framework/security/BUILD new file mode 100644 index 00000000000..16b1ffb58cd --- /dev/null +++ b/test/e2e/framework/security/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["apparmor.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/security", + visibility = ["//visibility:public"], + deps = [ + "//pkg/security/apparmor:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//test/e2e/framework:go_default_library", + "//test/e2e/framework/pod:go_default_library", + "//test/utils/image:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/common/apparmor.go b/test/e2e/framework/security/apparmor.go similarity index 78% rename from test/e2e/common/apparmor.go rename to test/e2e/framework/security/apparmor.go index 97e95435930..5f016baa904 100644 --- a/test/e2e/common/apparmor.go +++ b/test/e2e/framework/security/apparmor.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package common +package security import ( "fmt" @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -37,27 +38,17 @@ const ( loaderLabelValue = "e2e-apparmor-loader" ) -// AppArmorDistros are distros with AppArmor support -var AppArmorDistros = []string{"gci", "ubuntu"} - -func IsAppArmorSupported() bool { - return framework.NodeOSDistroIs(AppArmorDistros...) -} - -func SkipIfAppArmorNotSupported() { - framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...) -} - -func LoadAppArmorProfiles(f *framework.Framework) { - createAppArmorProfileCM(f) - createAppArmorProfileLoader(f) +// LoadAppArmorProfiles creates apparmor-profiles ConfigMap and apparmor-loader ReplicationController. +func LoadAppArmorProfiles(nsName string, clientset clientset.Interface) { + createAppArmorProfileCM(nsName, clientset) + createAppArmorProfileLoader(nsName, clientset) } // CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with // an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after // a single test, otherwise it will repeat the test every 1 second until failure. -func CreateAppArmorTestPod(f *framework.Framework, unconfined bool, runOnce bool) *v1.Pod { - profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name +func CreateAppArmorTestPod(nsName string, clientset clientset.Interface, podClient *framework.PodClient, unconfined bool, runOnce bool) *v1.Pod { + profile := "localhost/" + appArmorProfilePrefix + nsName testCmd := fmt.Sprintf(` if touch %[1]s; then echo "FAILURE: write to %[1]s should be denied" @@ -69,7 +60,7 @@ elif [[ $(< /proc/self/attr/current) != "%[3]s" ]]; then echo "FAILURE: not running with expected profile %[3]s" echo "found: $(cat /proc/self/attr/current)" exit 3 -fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name) +fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+nsName) if unconfined { profile = apparmor.ProfileNameUnconfined @@ -93,7 +84,7 @@ done`, testCmd) loaderAffinity := &v1.Affinity{ PodAffinity: &v1.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{ - Namespaces: []string{f.Namespace.Name}, + Namespaces: []string{nsName}, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{loaderLabelKey: loaderLabelValue}, }, @@ -124,26 +115,26 @@ done`, testCmd) } if runOnce { - pod = f.PodClient().Create(pod) + pod = podClient.Create(pod) framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace( - f.ClientSet, pod.Name, f.Namespace.Name)) + clientset, pod.Name, nsName)) var err error - pod, err = f.PodClient().Get(pod.Name, metav1.GetOptions{}) + pod, err = podClient.Get(pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err) } else { - pod = f.PodClient().CreateSync(pod) - framework.ExpectNoError(f.WaitForPodReady(pod.Name)) + pod = podClient.CreateSync(pod) + framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(clientset, pod.Name, nsName, framework.PodStartTimeout)) } // Verify Pod affinity colocated the Pods. - loader := getRunningLoaderPod(f) + loader := getRunningLoaderPod(nsName, clientset) framework.ExpectEqual(pod.Spec.NodeName, loader.Spec.NodeName) return pod } -func createAppArmorProfileCM(f *framework.Framework) { - profileName := appArmorProfilePrefix + f.Namespace.Name +func createAppArmorProfileCM(nsName string, clientset clientset.Interface) { + profileName := appArmorProfilePrefix + nsName profile := fmt.Sprintf(`#include profile %s flags=(attach_disconnected) { #include @@ -158,23 +149,23 @@ profile %s flags=(attach_disconnected) { cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "apparmor-profiles", - Namespace: f.Namespace.Name, + Namespace: nsName, }, Data: map[string]string{ profileName: profile, }, } - _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(cm) + _, err := clientset.CoreV1().ConfigMaps(nsName).Create(cm) framework.ExpectNoError(err, "Failed to create apparmor-profiles ConfigMap") } -func createAppArmorProfileLoader(f *framework.Framework) { +func createAppArmorProfileLoader(nsName string, clientset clientset.Interface) { True := true One := int32(1) loader := &v1.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ Name: "apparmor-loader", - Namespace: f.Namespace.Name, + Namespace: nsName, }, Spec: v1.ReplicationControllerSpec{ Replicas: &One, @@ -232,18 +223,18 @@ func createAppArmorProfileLoader(f *framework.Framework) { }, }, } - _, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(loader) + _, err := clientset.CoreV1().ReplicationControllers(nsName).Create(loader) framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController") // Wait for loader to be ready. - getRunningLoaderPod(f) + getRunningLoaderPod(nsName, clientset) } -func getRunningLoaderPod(f *framework.Framework) *v1.Pod { +func getRunningLoaderPod(nsName string, clientset clientset.Interface) *v1.Pod { label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue})) - pods, err := e2epod.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label) + pods, err := e2epod.WaitForPodsWithLabelScheduled(clientset, nsName, label) framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod") pod := &pods.Items[0] - framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(f.ClientSet, pod), "Failed to run apparmor-loader Pod") + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(clientset, pod), "Failed to run apparmor-loader Pod") return pod } diff --git a/test/e2e/framework/skip.go b/test/e2e/framework/skip.go index 631fc51deb8..23095eef047 100644 --- a/test/e2e/framework/skip.go +++ b/test/e2e/framework/skip.go @@ -174,3 +174,11 @@ func serverVersionGTE(v *utilversion.Version, c discovery.ServerVersionInterface } return sv.AtLeast(v), nil } + +// AppArmorDistros are distros with AppArmor support +var AppArmorDistros = []string{"gci", "ubuntu"} + +// SkipIfAppArmorNotSupported skips if the AppArmor is not supported by the node OS distro. +func SkipIfAppArmorNotSupported() { + SkipUnlessNodeOSDistroIs(AppArmorDistros...) +} diff --git a/test/e2e/node/BUILD b/test/e2e/node/BUILD index b0ed6391729..2b3fcb92605 100644 --- a/test/e2e/node/BUILD +++ b/test/e2e/node/BUILD @@ -38,13 +38,13 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/job:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/perf:go_default_library", "//test/e2e/framework/pod:go_default_library", + "//test/e2e/framework/security:go_default_library", "//test/e2e/framework/ssh:go_default_library", "//test/e2e/framework/volume:go_default_library", "//test/e2e/perftype:go_default_library", diff --git a/test/e2e/node/apparmor.go b/test/e2e/node/apparmor.go index d14818c386e..c003b28bfa8 100644 --- a/test/e2e/node/apparmor.go +++ b/test/e2e/node/apparmor.go @@ -17,8 +17,8 @@ limitations under the License. package node import ( - "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + e2esecurity "k8s.io/kubernetes/test/e2e/framework/security" "github.com/onsi/ginkgo" ) @@ -28,8 +28,8 @@ var _ = SIGDescribe("AppArmor", func() { ginkgo.Context("load AppArmor profiles", func() { ginkgo.BeforeEach(func() { - common.SkipIfAppArmorNotSupported() - common.LoadAppArmorProfiles(f) + framework.SkipIfAppArmorNotSupported() + e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet) }) ginkgo.AfterEach(func() { if !ginkgo.CurrentGinkgoTestDescription().Failed { @@ -39,11 +39,11 @@ var _ = SIGDescribe("AppArmor", func() { }) ginkgo.It("should enforce an AppArmor profile", func() { - common.CreateAppArmorTestPod(f, false, true) + e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true) }) ginkgo.It("can disable an AppArmor profile, using unconfined", func() { - common.CreateAppArmorTestPod(f, true, true) + e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), true, true) }) }) }) diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index 3fd237912d3..2a804fac4ad 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -34,11 +34,11 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/autoscaling:go_default_library", "//test/e2e/framework/job:go_default_library", "//test/e2e/framework/node:go_default_library", + "//test/e2e/framework/security:go_default_library", "//test/e2e/framework/service:go_default_library", "//test/e2e/framework/statefulset:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/upgrades/apparmor.go b/test/e2e/upgrades/apparmor.go index ae3dbef19ee..817fc4049b0 100644 --- a/test/e2e/upgrades/apparmor.go +++ b/test/e2e/upgrades/apparmor.go @@ -19,8 +19,8 @@ package upgrades import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + e2esecurity "k8s.io/kubernetes/test/e2e/framework/security" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -38,7 +38,7 @@ func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" } // Skip returns true when this test can be skipped. func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool { supportedImages := make(map[string]bool) - for _, d := range common.AppArmorDistros { + for _, d := range framework.AppArmorDistros { supportedImages[d] = true } @@ -53,11 +53,11 @@ func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool { // Setup creates a secret and then verifies that a pod can consume it. func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) { ginkgo.By("Loading AppArmor profiles to nodes") - common.LoadAppArmorProfiles(f) + e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet) // Create the initial test pod. ginkgo.By("Creating a long-running AppArmor enabled pod.") - t.pod = common.CreateAppArmorTestPod(f, false, false) + t.pod = e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, false) // Verify initial state. t.verifyNodesAppArmorEnabled(f) @@ -93,7 +93,7 @@ func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) { func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) { ginkgo.By("Verifying an AppArmor profile is enforced for a new pod") - common.CreateAppArmorTestPod(f, false, true) + e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true) } func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {