From 13558e3fe7c9449ff489a373c3181ca4e5a2fb1b Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Tue, 5 Sep 2017 19:19:54 -0700 Subject: [PATCH 1/3] Fix AppArmor test at scale --- test/e2e/common/apparmor.go | 51 +++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/test/e2e/common/apparmor.go b/test/e2e/common/apparmor.go index efe8716222d..a0a05b20288 100644 --- a/test/e2e/common/apparmor.go +++ b/test/e2e/common/apparmor.go @@ -20,8 +20,8 @@ import ( "fmt" api "k8s.io/api/core/v1" - extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/test/e2e/framework" ) @@ -30,6 +30,9 @@ const ( appArmorProfilePrefix = "e2e-apparmor-test-" appArmorAllowedPath = "/expect_allowed_write" appArmorDeniedPath = "/expect_permission_denied" + + loaderLabelKey = "name" + loaderLabelValue = "e2e-apparmor-loader" ) // AppArmorDistros are distros with AppArmor support @@ -40,10 +43,8 @@ func SkipIfAppArmorNotSupported() { } func LoadAppArmorProfiles(f *framework.Framework) { - _, err := createAppArmorProfileCM(f) - framework.ExpectNoError(err) - _, err = createAppArmorProfileLoader(f) - framework.ExpectNoError(err) + createAppArmorProfileCM(f) + createAppArmorProfileLoader(f) } // CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with @@ -71,6 +72,18 @@ sleep 1 done`, testCmd) } + loaderAffinity := &api.Affinity{ + PodAffinity: &api.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{{ + Namespaces: []string{f.Namespace.Name}, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{loaderLabelKey: loaderLabelValue}, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + } + pod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-apparmor-", @@ -82,6 +95,7 @@ done`, testCmd) }, }, Spec: api.PodSpec{ + Affinity: loaderAffinity, Containers: []api.Container{{ Name: "test", Image: busyboxImage, @@ -103,7 +117,7 @@ done`, testCmd) return pod } -func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) { +func createAppArmorProfileCM(f *framework.Framework) { profileName := appArmorProfilePrefix + f.Namespace.Name profile := fmt.Sprintf(`#include profile %s flags=(attach_disconnected) { @@ -125,21 +139,23 @@ profile %s flags=(attach_disconnected) { profileName: profile, }, } - return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) + _, err := f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) + framework.ExpectNoError(err, "Failed to create apparmor-profiles ConfigMap") } -func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) { +func createAppArmorProfileLoader(f *framework.Framework) { True := true - // Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml - loader := &extensions.DaemonSet{ + One := int32(1) + loader := &api.ReplicationController{ ObjectMeta: metav1.ObjectMeta{ Name: "apparmor-loader", Namespace: f.Namespace.Name, }, - Spec: extensions.DaemonSetSpec{ - Template: api.PodTemplateSpec{ + Spec: api.ReplicationControllerSpec{ + Replicas: &One, + Template: &api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"name": "apparmor-loader"}, + Labels: map[string]string{loaderLabelKey: loaderLabelValue}, }, Spec: api.PodSpec{ Containers: []api.Container{{ @@ -191,5 +207,12 @@ func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, }, }, } - return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader) + _, err := f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Create(loader) + framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController") + + // Wait for loader to be ready. + label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue})) + pods, err := framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label) + framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod") + framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, &pods.Items[0]), "Failed to run apparmor-loader Pod") } From 7d180ce0098cb2c14bf310c1f5cfdce2044aabcc Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Thu, 7 Sep 2017 13:12:50 -0700 Subject: [PATCH 2/3] Verify that AppArmor pod is colocated with the loader --- test/e2e/common/apparmor.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/e2e/common/apparmor.go b/test/e2e/common/apparmor.go index a0a05b20288..a81eea8349b 100644 --- a/test/e2e/common/apparmor.go +++ b/test/e2e/common/apparmor.go @@ -24,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/gomega" ) const ( @@ -109,11 +111,18 @@ done`, testCmd) pod = f.PodClient().Create(pod) framework.ExpectNoError(framework.WaitForPodSuccessInNamespace( f.ClientSet, pod.Name, f.Namespace.Name)) + var err error + pod, err = f.PodClient().Get(pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) } else { pod = f.PodClient().CreateSync(pod) framework.ExpectNoError(f.WaitForPodReady(pod.Name)) } + // Verify Pod affinity colocated the Pods. + loader := getRunningLoaderPod(f) + Expect(pod.Spec.NodeName).To(Equal(loader.Spec.NodeName)) + return pod } @@ -211,8 +220,14 @@ func createAppArmorProfileLoader(f *framework.Framework) { framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController") // Wait for loader to be ready. + getRunningLoaderPod(f) +} + +func getRunningLoaderPod(f *framework.Framework) *api.Pod { label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue})) pods, err := framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label) framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod") - framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, &pods.Items[0]), "Failed to run apparmor-loader Pod") + pod := &pods.Items[0] + framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod), "Failed to run apparmor-loader Pod") + return pod } From 2604f6760ac401a5dc56a214ed60481c46578745 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Thu, 7 Sep 2017 13:13:49 -0700 Subject: [PATCH 3/3] Rerun hack/update-bazel.sh --- test/e2e/common/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index 72fb4670550..d599a2c6dcb 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -50,7 +50,6 @@ go_library( "//vendor/golang.org/x/net/websocket:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",