AppArmor cluster upgrade test

test/e2e/BUILD

@@ -143,7 +143,6 @@ go_library(
         "//pkg/master/ports:go_default_library",
         "//pkg/metrics:go_default_library",
         "//pkg/quota/evaluator/core:go_default_library",
-        "//pkg/security/apparmor:go_default_library",
         "//pkg/util:go_default_library",
         "//pkg/util/exec:go_default_library",
         "//pkg/util/logs:go_default_library",

test/e2e/apparmor.go

@@ -17,162 +17,22 @@ limitations under the License.
 package e2e
 
 import (
-	"fmt"
+	"k8s.io/kubernetes/test/e2e/common"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	api "k8s.io/kubernetes/pkg/api/v1"
-	extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
-	"k8s.io/kubernetes/pkg/security/apparmor"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
 )
 
-const (
-	profilePrefix = "e2e-apparmor-test-"
-	allowedPath   = "/expect_allowed_write"
-	deniedPath    = "/expect_permission_denied"
-)
-
 var _ = framework.KubeDescribe("AppArmor", func() {
 	f := framework.NewDefaultFramework("apparmor")
 
 	BeforeEach(func() {
-		SkipIfAppArmorNotSupported()
+		common.SkipIfAppArmorNotSupported()
-		LoadAppArmorProfiles(f)
+		common.LoadAppArmorProfiles(f)
 	})
 
 	It("should enforce an AppArmor profile", func() {
-		profile := "localhost/" + profilePrefix + f.Namespace.Name
+		common.CreateAppArmorTestPod(f, true)
-		testCmd := fmt.Sprintf(`
-if touch %[1]s; then
-  echo "FAILURE: write to %[1]s should be denied"
-  exit 1
-elif ! touch %[2]s; then
-  echo "FAILURE: write to %[2]s should be allowed"
-  exit 2
-fi`, deniedPath, allowedPath)
-		pod := &api.Pod{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "test-apparmor",
-				Annotations: map[string]string{
-					apparmor.ContainerAnnotationKeyPrefix + "test": profile,
-				},
-			},
-			Spec: api.PodSpec{
-				Containers: []api.Container{{
-					Name:    "test",
-					Image:   "gcr.io/google_containers/busybox:1.24",
-					Command: []string{"sh", "-c", testCmd},
-				}},
-				RestartPolicy: api.RestartPolicyNever,
-			},
-		}
-		f.PodClient().Create(pod)
-		framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(
-			f.ClientSet, pod.Name, f.Namespace.Name))
 		framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
 	})
 })
-
-func SkipIfAppArmorNotSupported() {
-	framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
-}
-
-func LoadAppArmorProfiles(f *framework.Framework) {
-	_, err := createAppArmorProfileCM(f)
-	framework.ExpectNoError(err)
-	_, err = createAppArmorProfileLoader(f)
-	framework.ExpectNoError(err)
-}
-
-func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) {
-	profileName := profilePrefix + f.Namespace.Name
-	profile := fmt.Sprintf(`#include <tunables/global>
-profile %s flags=(attach_disconnected) {
-  #include <abstractions/base>
-
-  file,
-
-  deny %s w,
-  audit %s w,
-}
-`, profileName, deniedPath, allowedPath)
-
-	cm := &api.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "apparmor-profiles",
-			Namespace: f.Namespace.Name,
-		},
-		Data: map[string]string{
-			profileName: profile,
-		},
-	}
-	return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm)
-}
-
-func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) {
-	True := true
-	// Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml
-	loader := &extensions.DaemonSet{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "apparmor-loader",
-			Namespace: f.Namespace.Name,
-		},
-		Spec: extensions.DaemonSetSpec{
-			Template: api.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{"name": "apparmor-loader"},
-				},
-				Spec: api.PodSpec{
-					Containers: []api.Container{{
-						Name:  "apparmor-loader",
-						Image: "gcr.io/google_containers/apparmor-loader:0.1",
-						Args:  []string{"-poll", "10s", "/profiles"},
-						SecurityContext: &api.SecurityContext{
-							Privileged: &True,
-						},
-						VolumeMounts: []api.VolumeMount{{
-							Name:      "sys",
-							MountPath: "/sys",
-							ReadOnly:  true,
-						}, {
-							Name:      "apparmor-includes",
-							MountPath: "/etc/apparmor.d",
-							ReadOnly:  true,
-						}, {
-							Name:      "profiles",
-							MountPath: "/profiles",
-							ReadOnly:  true,
-						}},
-					}},
-					Volumes: []api.Volume{{
-						Name: "sys",
-						VolumeSource: api.VolumeSource{
-							HostPath: &api.HostPathVolumeSource{
-								Path: "/sys",
-							},
-						},
-					}, {
-						Name: "apparmor-includes",
-						VolumeSource: api.VolumeSource{
-							HostPath: &api.HostPathVolumeSource{
-								Path: "/etc/apparmor.d",
-							},
-						},
-					}, {
-						Name: "profiles",
-						VolumeSource: api.VolumeSource{
-							ConfigMap: &api.ConfigMapVolumeSource{
-								LocalObjectReference: api.LocalObjectReference{
-									Name: "apparmor-profiles",
-								},
-							},
-						},
-					}},
-				},
-			},
-		},
-	}
-	return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader)
-}

test/e2e/cluster_upgrade.go

@@ -40,6 +40,7 @@ var upgradeTests = []upgrades.Test{
 	&upgrades.PersistentVolumeUpgradeTest{},
 	&upgrades.DaemonSetUpgradeTest{},
 	&upgrades.IngressUpgradeTest{},
+	&upgrades.AppArmorUpgradeTest{},
 }
 
 var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {

test/e2e/common/BUILD

@@ -10,6 +10,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "apparmor.go",
         "autoscaling_utils.go",
         "configmap.go",
         "container_probe.go",
@@ -36,11 +37,13 @@ go_library(
         "//pkg/api/v1:go_default_library",
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/apis/autoscaling/v1:go_default_library",
+        "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/conditions:go_default_library",
         "//pkg/kubelet:go_default_library",
         "//pkg/kubelet/sysctl:go_default_library",
+        "//pkg/security/apparmor:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor:github.com/golang/glog",

test/e2e/common/apparmor.go

@@ -0,0 +1,191 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	api "k8s.io/kubernetes/pkg/api/v1"
+	extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
+	"k8s.io/kubernetes/pkg/security/apparmor"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	appArmorProfilePrefix = "e2e-apparmor-test-"
+	appArmorAllowedPath   = "/expect_allowed_write"
+	appArmorDeniedPath    = "/expect_permission_denied"
+)
+
+func SkipIfAppArmorNotSupported() {
+	framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
+}
+
+func LoadAppArmorProfiles(f *framework.Framework) {
+	_, err := createAppArmorProfileCM(f)
+	framework.ExpectNoError(err)
+	_, err = createAppArmorProfileLoader(f)
+	framework.ExpectNoError(err)
+}
+
+// CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with
+// an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after
+// a single test, otherwise it will repeat the test every 1 second until failure.
+func CreateAppArmorTestPod(f *framework.Framework, runOnce bool) *api.Pod {
+	profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name
+	testCmd := fmt.Sprintf(`
+if touch %[1]s; then
+  echo "FAILURE: write to %[1]s should be denied"
+  exit 1
+elif ! touch %[2]s; then
+  echo "FAILURE: write to %[2]s should be allowed"
+  exit 2
+elif ! grep "%[3]s" /proc/1/attr/current; then
+  echo "FAILURE: not running with expected profile %[3]s"
+  exit 3
+fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name)
+
+	if !runOnce {
+		testCmd = fmt.Sprintf(`while true; do
+%s
+sleep 1
+done`, testCmd)
+	}
+
+	pod := &api.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "test-apparmor-",
+			Annotations: map[string]string{
+				apparmor.ContainerAnnotationKeyPrefix + "test": profile,
+			},
+			Labels: map[string]string{
+				"test": "apparmor",
+			},
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{{
+				Name:    "test",
+				Image:   "gcr.io/google_containers/busybox:1.24",
+				Command: []string{"sh", "-c", testCmd},
+			}},
+			RestartPolicy: api.RestartPolicyNever,
+		},
+	}
+
+	if runOnce {
+		pod = f.PodClient().Create(pod)
+		framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(
+			f.ClientSet, pod.Name, f.Namespace.Name))
+	} else {
+		pod = f.PodClient().CreateSync(pod)
+		framework.ExpectNoError(f.WaitForPodReady(pod.Name))
+	}
+
+	return pod
+}
+
+func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) {
+	profileName := appArmorProfilePrefix + f.Namespace.Name
+	profile := fmt.Sprintf(`#include <tunables/global>
+profile %s flags=(attach_disconnected) {
+  #include <abstractions/base>
+
+  file,
+
+  deny %s w,
+  audit %s w,
+}
+`, profileName, appArmorDeniedPath, appArmorAllowedPath)
+
+	cm := &api.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "apparmor-profiles",
+			Namespace: f.Namespace.Name,
+		},
+		Data: map[string]string{
+			profileName: profile,
+		},
+	}
+	return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm)
+}
+
+func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) {
+	True := true
+	// Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml
+	loader := &extensions.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "apparmor-loader",
+			Namespace: f.Namespace.Name,
+		},
+		Spec: extensions.DaemonSetSpec{
+			Template: api.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": "apparmor-loader"},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{
+						Name:  "apparmor-loader",
+						Image: "gcr.io/google_containers/apparmor-loader:0.1",
+						Args:  []string{"-poll", "10s", "/profiles"},
+						SecurityContext: &api.SecurityContext{
+							Privileged: &True,
+						},
+						VolumeMounts: []api.VolumeMount{{
+							Name:      "sys",
+							MountPath: "/sys",
+							ReadOnly:  true,
+						}, {
+							Name:      "apparmor-includes",
+							MountPath: "/etc/apparmor.d",
+							ReadOnly:  true,
+						}, {
+							Name:      "profiles",
+							MountPath: "/profiles",
+							ReadOnly:  true,
+						}},
+					}},
+					Volumes: []api.Volume{{
+						Name: "sys",
+						VolumeSource: api.VolumeSource{
+							HostPath: &api.HostPathVolumeSource{
+								Path: "/sys",
+							},
+						},
+					}, {
+						Name: "apparmor-includes",
+						VolumeSource: api.VolumeSource{
+							HostPath: &api.HostPathVolumeSource{
+								Path: "/etc/apparmor.d",
+							},
+						},
+					}, {
+						Name: "profiles",
+						VolumeSource: api.VolumeSource{
+							ConfigMap: &api.ConfigMapVolumeSource{
+								LocalObjectReference: api.LocalObjectReference{
+									Name: "apparmor-profiles",
+								},
+							},
+						},
+					}},
+				},
+			},
+		},
+	}
+	return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader)
+}

test/e2e/upgrades/BUILD

@@ -10,6 +10,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "apparmor.go",
         "configmaps.go",
         "daemonsets.go",
         "deployments.go",
@@ -36,6 +37,7 @@ go_library(
         "//test/e2e/framework:go_default_library",
         "//vendor:github.com/onsi/ginkgo",
         "//vendor:github.com/onsi/gomega",
+        "//vendor:github.com/onsi/gomega/gstruct",
         "//vendor:k8s.io/apimachinery/pkg/api/errors",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/labels",

test/e2e/upgrades/apparmor.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrades
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	api "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/test/e2e/common"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+)
+
+// AppArmorUpgradeTest tests that AppArmor profiles are enforced & usable across upgrades.
+type AppArmorUpgradeTest struct {
+	pod *api.Pod
+}
+
+func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
+
+// Setup creates a secret and then verifies that a pod can consume it.
+func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
+	common.SkipIfAppArmorNotSupported()
+	By("Loading AppArmor profiles to nodes")
+	common.LoadAppArmorProfiles(f)
+
+	// Create the initial test pod.
+	By("Creating a long-running AppArmor enabled pod.")
+	t.pod = common.CreateAppArmorTestPod(f, false)
+
+	// Verify initial state.
+	t.verifyNodesAppArmorEnabled(f)
+	t.verifyNewPodSucceeds(f)
+}
+
+// Test waits for the upgrade to complete, and then verifies that a
+// pod can still consume the secret.
+func (t *AppArmorUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
+	<-done
+	if upgrade == MasterUpgrade {
+		t.verifyPodStillUp(f)
+	}
+	t.verifyNodesAppArmorEnabled(f)
+	t.verifyNewPodSucceeds(f)
+}
+
+// Teardown cleans up any remaining resources.
+func (t *AppArmorUpgradeTest) Teardown(f *framework.Framework) {
+	// rely on the namespace deletion to clean up everything
+	By("Logging container failures")
+	framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
+}
+
+func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
+	By("Verifying an AppArmor profile is continuously enforced for a pod")
+	pod, err := f.PodClient().Get(t.pod.Name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "Should be able to get pod")
+	Expect(pod.Status.Phase).To(Equal(api.PodRunning), "Pod should stay running")
+	Expect(pod.Status.ContainerStatuses[0].State.Running).NotTo(BeNil(), "Container should be running")
+	Expect(pod.Status.ContainerStatuses[0].RestartCount).To(BeZero(), "Container should not need to be restarted")
+}
+
+func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
+	By("Verifying an AppArmor profile is enforced for a new pod")
+	common.CreateAppArmorTestPod(f, true)
+}
+
+func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {
+	By("Verifying nodes are AppArmor enabled")
+	nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
+	framework.ExpectNoError(err, "Failed to list nodes")
+	for _, node := range nodes.Items {
+		Expect(node.Status.Conditions).To(gstruct.MatchElements(conditionType, gstruct.IgnoreExtras, gstruct.Elements{
+			"Ready": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+				"Message": ContainSubstring("AppArmor enabled"),
+			}),
+		}))
+	}
+}
+
+func conditionType(condition interface{}) string {
+	return string(condition.(api.NodeCondition).Type)
+}