Merge pull request #84439 from danielqsj/f-apparmor

refactor apparmor utils in e2e

test/e2e/auth/BUILD

@@ -51,7 +51,6 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
-        "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/auth:go_default_library",
         "//test/e2e/framework/deployment:go_default_library",

test/e2e/auth/pod_security_policy.go

@@ -31,7 +31,6 @@ import (
 	"k8s.io/kubernetes/pkg/security/apparmor"
 	"k8s.io/kubernetes/pkg/security/podsecuritypolicy/seccomp"
 	psputil "k8s.io/kubernetes/pkg/security/podsecuritypolicy/util"
-	"k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/auth"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -168,7 +167,7 @@ func testPrivilegedPods(tester func(pod *v1.Pod)) {
 		tester(hostipc)
 	})
 
-	if common.IsAppArmorSupported() {
+	if framework.IsAppArmorSupported() {
 		ginkgo.By("Running a custom AppArmor profile pod", func() {
 			aa := restrictedPod("apparmor")
 			// Every node is expected to have the docker-default profile.

test/e2e/common/BUILD

@@ -8,7 +8,6 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
-        "apparmor.go",
         "configmap.go",
         "configmap_volume.go",
         "container.go",
@@ -51,7 +50,6 @@ go_library(
         "//pkg/kubelet/images:go_default_library",
         "//pkg/kubelet/runtimeclass/testing:go_default_library",
         "//pkg/kubelet/sysctl:go_default_library",
-        "//pkg/security/apparmor:go_default_library",
         "//staging/src/k8s.io/api/coordination/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",

test/e2e/framework/BUILD

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "checks.go",
         "cleanup.go",
         "exec_util.go",
         "expect.go",
@@ -133,6 +134,7 @@ filegroup(
         "//test/e2e/framework/pv:all-srcs",
         "//test/e2e/framework/replicaset:all-srcs",
         "//test/e2e/framework/resource:all-srcs",
+        "//test/e2e/framework/security:all-srcs",
         "//test/e2e/framework/service:all-srcs",
         "//test/e2e/framework/ssh:all-srcs",
         "//test/e2e/framework/statefulset:all-srcs",

test/e2e/framework/checks.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+// IsAppArmorSupported checks whether the AppArmor is supported by the node OS distro.
+func IsAppArmorSupported() bool {
+	return NodeOSDistroIs(AppArmorDistros...)
+}

test/e2e/framework/security/BUILD

@@ -0,0 +1,32 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["apparmor.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/framework/security",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/security/apparmor:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/pod:go_default_library",
+        "//test/utils/image:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/framework/security/apparmor.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package common
+package security
 
 import (
 	"fmt"
@@ -22,6 +22,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/security/apparmor"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -37,27 +38,17 @@ const (
 	loaderLabelValue = "e2e-apparmor-loader"
 )
 
-// AppArmorDistros are distros with AppArmor support
+// LoadAppArmorProfiles creates apparmor-profiles ConfigMap and apparmor-loader ReplicationController.
-var AppArmorDistros = []string{"gci", "ubuntu"}
+func LoadAppArmorProfiles(nsName string, clientset clientset.Interface) {
-
+	createAppArmorProfileCM(nsName, clientset)
-func IsAppArmorSupported() bool {
+	createAppArmorProfileLoader(nsName, clientset)
-	return framework.NodeOSDistroIs(AppArmorDistros...)
-}
-
-func SkipIfAppArmorNotSupported() {
-	framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...)
-}
-
-func LoadAppArmorProfiles(f *framework.Framework) {
-	createAppArmorProfileCM(f)
-	createAppArmorProfileLoader(f)
 }
 
 // CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with
 // an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after
 // a single test, otherwise it will repeat the test every 1 second until failure.
-func CreateAppArmorTestPod(f *framework.Framework, unconfined bool, runOnce bool) *v1.Pod {
+func CreateAppArmorTestPod(nsName string, clientset clientset.Interface, podClient *framework.PodClient, unconfined bool, runOnce bool) *v1.Pod {
-	profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name
+	profile := "localhost/" + appArmorProfilePrefix + nsName
 	testCmd := fmt.Sprintf(`
 if touch %[1]s; then
   echo "FAILURE: write to %[1]s should be denied"
@@ -69,7 +60,7 @@ elif [[ $(< /proc/self/attr/current) != "%[3]s" ]]; then
   echo "FAILURE: not running with expected profile %[3]s"
   echo "found: $(cat /proc/self/attr/current)"
   exit 3
-fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name)
+fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+nsName)
 
 	if unconfined {
 		profile = apparmor.ProfileNameUnconfined
@@ -93,7 +84,7 @@ done`, testCmd)
 	loaderAffinity := &v1.Affinity{
 		PodAffinity: &v1.PodAffinity{
 			RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{
-				Namespaces: []string{f.Namespace.Name},
+				Namespaces: []string{nsName},
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{loaderLabelKey: loaderLabelValue},
 				},
@@ -124,26 +115,26 @@ done`, testCmd)
 	}
 
 	if runOnce {
-		pod = f.PodClient().Create(pod)
+		pod = podClient.Create(pod)
 		framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(
-			f.ClientSet, pod.Name, f.Namespace.Name))
+			clientset, pod.Name, nsName))
 		var err error
-		pod, err = f.PodClient().Get(pod.Name, metav1.GetOptions{})
+		pod, err = podClient.Get(pod.Name, metav1.GetOptions{})
 		framework.ExpectNoError(err)
 	} else {
-		pod = f.PodClient().CreateSync(pod)
+		pod = podClient.CreateSync(pod)
-		framework.ExpectNoError(f.WaitForPodReady(pod.Name))
+		framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(clientset, pod.Name, nsName, framework.PodStartTimeout))
 	}
 
 	// Verify Pod affinity colocated the Pods.
-	loader := getRunningLoaderPod(f)
+	loader := getRunningLoaderPod(nsName, clientset)
 	framework.ExpectEqual(pod.Spec.NodeName, loader.Spec.NodeName)
 
 	return pod
 }
 
-func createAppArmorProfileCM(f *framework.Framework) {
+func createAppArmorProfileCM(nsName string, clientset clientset.Interface) {
-	profileName := appArmorProfilePrefix + f.Namespace.Name
+	profileName := appArmorProfilePrefix + nsName
 	profile := fmt.Sprintf(`#include <tunables/global>
 profile %s flags=(attach_disconnected) {
   #include <abstractions/base>
@@ -158,23 +149,23 @@ profile %s flags=(attach_disconnected) {
 	cm := &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "apparmor-profiles",
-			Namespace: f.Namespace.Name,
+			Namespace: nsName,
 		},
 		Data: map[string]string{
 			profileName: profile,
 		},
 	}
-	_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(cm)
+	_, err := clientset.CoreV1().ConfigMaps(nsName).Create(cm)
 	framework.ExpectNoError(err, "Failed to create apparmor-profiles ConfigMap")
 }
 
-func createAppArmorProfileLoader(f *framework.Framework) {
+func createAppArmorProfileLoader(nsName string, clientset clientset.Interface) {
 	True := true
 	One := int32(1)
 	loader := &v1.ReplicationController{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "apparmor-loader",
-			Namespace: f.Namespace.Name,
+			Namespace: nsName,
 		},
 		Spec: v1.ReplicationControllerSpec{
 			Replicas: &One,
@@ -232,18 +223,18 @@ func createAppArmorProfileLoader(f *framework.Framework) {
 			},
 		},
 	}
-	_, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(loader)
+	_, err := clientset.CoreV1().ReplicationControllers(nsName).Create(loader)
 	framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController")
 
 	// Wait for loader to be ready.
-	getRunningLoaderPod(f)
+	getRunningLoaderPod(nsName, clientset)
 }
 
-func getRunningLoaderPod(f *framework.Framework) *v1.Pod {
+func getRunningLoaderPod(nsName string, clientset clientset.Interface) *v1.Pod {
 	label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue}))
-	pods, err := e2epod.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label)
+	pods, err := e2epod.WaitForPodsWithLabelScheduled(clientset, nsName, label)
 	framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod")
 	pod := &pods.Items[0]
-	framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(f.ClientSet, pod), "Failed to run apparmor-loader Pod")
+	framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(clientset, pod), "Failed to run apparmor-loader Pod")
 	return pod
 }

test/e2e/framework/skip.go

@@ -174,3 +174,11 @@ func serverVersionGTE(v *utilversion.Version, c discovery.ServerVersionInterface
 	}
 	return sv.AtLeast(v), nil
 }
+
+// AppArmorDistros are distros with AppArmor support
+var AppArmorDistros = []string{"gci", "ubuntu"}
+
+// SkipIfAppArmorNotSupported skips if the AppArmor is not supported by the node OS distro.
+func SkipIfAppArmorNotSupported() {
+	SkipUnlessNodeOSDistroIs(AppArmorDistros...)
+}

test/e2e/node/BUILD

@@ -38,13 +38,13 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
-        "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/job:go_default_library",
         "//test/e2e/framework/kubelet:go_default_library",
         "//test/e2e/framework/node:go_default_library",
         "//test/e2e/framework/perf:go_default_library",
         "//test/e2e/framework/pod:go_default_library",
+        "//test/e2e/framework/security:go_default_library",
         "//test/e2e/framework/ssh:go_default_library",
         "//test/e2e/framework/volume:go_default_library",
         "//test/e2e/perftype:go_default_library",

test/e2e/node/apparmor.go

@@ -17,8 +17,8 @@ limitations under the License.
 package node
 
 import (
-	"k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2esecurity "k8s.io/kubernetes/test/e2e/framework/security"
 
 	"github.com/onsi/ginkgo"
 )
@@ -28,8 +28,8 @@ var _ = SIGDescribe("AppArmor", func() {
 
 	ginkgo.Context("load AppArmor profiles", func() {
 		ginkgo.BeforeEach(func() {
-			common.SkipIfAppArmorNotSupported()
+			framework.SkipIfAppArmorNotSupported()
-			common.LoadAppArmorProfiles(f)
+			e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet)
 		})
 		ginkgo.AfterEach(func() {
 			if !ginkgo.CurrentGinkgoTestDescription().Failed {
@@ -39,11 +39,11 @@ var _ = SIGDescribe("AppArmor", func() {
 		})
 
 		ginkgo.It("should enforce an AppArmor profile", func() {
-			common.CreateAppArmorTestPod(f, false, true)
+			e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true)
 		})
 
 		ginkgo.It("can disable an AppArmor profile, using unconfined", func() {
-			common.CreateAppArmorTestPod(f, true, true)
+			e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), true, true)
 		})
 	})
 })

test/e2e/upgrades/BUILD

@@ -34,11 +34,11 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
-        "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/autoscaling:go_default_library",
         "//test/e2e/framework/job:go_default_library",
         "//test/e2e/framework/node:go_default_library",
+        "//test/e2e/framework/security:go_default_library",
         "//test/e2e/framework/service:go_default_library",
         "//test/e2e/framework/statefulset:go_default_library",
         "//test/e2e/framework/testfiles:go_default_library",

test/e2e/upgrades/apparmor.go

@@ -19,8 +19,8 @@ package upgrades
 import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2esecurity "k8s.io/kubernetes/test/e2e/framework/security"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -38,7 +38,7 @@ func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
 // Skip returns true when this test can be skipped.
 func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
 	supportedImages := make(map[string]bool)
-	for _, d := range common.AppArmorDistros {
+	for _, d := range framework.AppArmorDistros {
 		supportedImages[d] = true
 	}
 
@@ -53,11 +53,11 @@ func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
 // Setup creates a secret and then verifies that a pod can consume it.
 func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
 	ginkgo.By("Loading AppArmor profiles to nodes")
-	common.LoadAppArmorProfiles(f)
+	e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet)
 
 	// Create the initial test pod.
 	ginkgo.By("Creating a long-running AppArmor enabled pod.")
-	t.pod = common.CreateAppArmorTestPod(f, false, false)
+	t.pod = e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, false)
 
 	// Verify initial state.
 	t.verifyNodesAppArmorEnabled(f)
@@ -93,7 +93,7 @@ func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
 
 func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
 	ginkgo.By("Verifying an AppArmor profile is enforced for a new pod")
-	common.CreateAppArmorTestPod(f, false, true)
+	e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true)
 }
 
 func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {