diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 7b1240816cd..564559a8992 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -144,7 +144,6 @@ go_library( "//pkg/master/ports:go_default_library", "//pkg/metrics:go_default_library", "//pkg/quota/evaluator/core:go_default_library", - "//pkg/security/apparmor:go_default_library", "//pkg/util:go_default_library", "//pkg/util/exec:go_default_library", "//pkg/util/logs:go_default_library", diff --git a/test/e2e/apparmor.go b/test/e2e/apparmor.go index 6a61d65f627..389d549045d 100644 --- a/test/e2e/apparmor.go +++ b/test/e2e/apparmor.go @@ -17,162 +17,22 @@ limitations under the License. package e2e import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "k8s.io/kubernetes/pkg/api/v1" - extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" - "k8s.io/kubernetes/pkg/security/apparmor" + "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" ) -const ( - profilePrefix = "e2e-apparmor-test-" - allowedPath = "/expect_allowed_write" - deniedPath = "/expect_permission_denied" -) - var _ = framework.KubeDescribe("AppArmor", func() { f := framework.NewDefaultFramework("apparmor") BeforeEach(func() { - SkipIfAppArmorNotSupported() - LoadAppArmorProfiles(f) + common.SkipIfAppArmorNotSupported() + common.LoadAppArmorProfiles(f) }) It("should enforce an AppArmor profile", func() { - profile := "localhost/" + profilePrefix + f.Namespace.Name - testCmd := fmt.Sprintf(` -if touch %[1]s; then - echo "FAILURE: write to %[1]s should be denied" - exit 1 -elif ! touch %[2]s; then - echo "FAILURE: write to %[2]s should be allowed" - exit 2 -fi`, deniedPath, allowedPath) - pod := &api.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-apparmor", - Annotations: map[string]string{ - apparmor.ContainerAnnotationKeyPrefix + "test": profile, - }, - }, - Spec: api.PodSpec{ - Containers: []api.Container{{ - Name: "test", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"sh", "-c", testCmd}, - }}, - RestartPolicy: api.RestartPolicyNever, - }, - } - f.PodClient().Create(pod) - framework.ExpectNoError(framework.WaitForPodSuccessInNamespace( - f.ClientSet, pod.Name, f.Namespace.Name)) + common.CreateAppArmorTestPod(f, true) framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) }) }) - -func SkipIfAppArmorNotSupported() { - framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu") -} - -func LoadAppArmorProfiles(f *framework.Framework) { - _, err := createAppArmorProfileCM(f) - framework.ExpectNoError(err) - _, err = createAppArmorProfileLoader(f) - framework.ExpectNoError(err) -} - -func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) { - profileName := profilePrefix + f.Namespace.Name - profile := fmt.Sprintf(`#include -profile %s flags=(attach_disconnected) { - #include - - file, - - deny %s w, - audit %s w, -} -`, profileName, deniedPath, allowedPath) - - cm := &api.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "apparmor-profiles", - Namespace: f.Namespace.Name, - }, - Data: map[string]string{ - profileName: profile, - }, - } - return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) -} - -func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) { - True := true - // Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml - loader := &extensions.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "apparmor-loader", - Namespace: f.Namespace.Name, - }, - Spec: extensions.DaemonSetSpec{ - Template: api.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"name": "apparmor-loader"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{{ - Name: "apparmor-loader", - Image: "gcr.io/google_containers/apparmor-loader:0.1", - Args: []string{"-poll", "10s", "/profiles"}, - SecurityContext: &api.SecurityContext{ - Privileged: &True, - }, - VolumeMounts: []api.VolumeMount{{ - Name: "sys", - MountPath: "/sys", - ReadOnly: true, - }, { - Name: "apparmor-includes", - MountPath: "/etc/apparmor.d", - ReadOnly: true, - }, { - Name: "profiles", - MountPath: "/profiles", - ReadOnly: true, - }}, - }}, - Volumes: []api.Volume{{ - Name: "sys", - VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, { - Name: "apparmor-includes", - VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - }, - }, - }, { - Name: "profiles", - VolumeSource: api.VolumeSource{ - ConfigMap: &api.ConfigMapVolumeSource{ - LocalObjectReference: api.LocalObjectReference{ - Name: "apparmor-profiles", - }, - }, - }, - }}, - }, - }, - }, - } - return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader) -} diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 1adbabce7b5..136ba5ddfba 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -35,6 +35,7 @@ var upgradeTests = []upgrades.Test{ &upgrades.PersistentVolumeUpgradeTest{}, &upgrades.DaemonSetUpgradeTest{}, &upgrades.IngressUpgradeTest{}, + &upgrades.AppArmorUpgradeTest{}, } var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index baaf0072e74..b303dcf9314 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -10,6 +10,7 @@ load( go_library( name = "go_default_library", srcs = [ + "apparmor.go", "autoscaling_utils.go", "configmap.go", "container_probe.go", @@ -36,11 +37,13 @@ go_library( "//pkg/api/v1:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/apis/autoscaling/v1:go_default_library", + "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/conditions:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/sysctl:go_default_library", + "//pkg/security/apparmor:go_default_library", "//test/e2e/framework:go_default_library", "//test/utils:go_default_library", "//vendor:github.com/golang/glog", diff --git a/test/e2e/common/apparmor.go b/test/e2e/common/apparmor.go new file mode 100644 index 00000000000..914f0daf8cf --- /dev/null +++ b/test/e2e/common/apparmor.go @@ -0,0 +1,191 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + api "k8s.io/kubernetes/pkg/api/v1" + extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + "k8s.io/kubernetes/pkg/security/apparmor" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + appArmorProfilePrefix = "e2e-apparmor-test-" + appArmorAllowedPath = "/expect_allowed_write" + appArmorDeniedPath = "/expect_permission_denied" +) + +func SkipIfAppArmorNotSupported() { + framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu") +} + +func LoadAppArmorProfiles(f *framework.Framework) { + _, err := createAppArmorProfileCM(f) + framework.ExpectNoError(err) + _, err = createAppArmorProfileLoader(f) + framework.ExpectNoError(err) +} + +// CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with +// an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after +// a single test, otherwise it will repeat the test every 1 second until failure. +func CreateAppArmorTestPod(f *framework.Framework, runOnce bool) *api.Pod { + profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name + testCmd := fmt.Sprintf(` +if touch %[1]s; then + echo "FAILURE: write to %[1]s should be denied" + exit 1 +elif ! touch %[2]s; then + echo "FAILURE: write to %[2]s should be allowed" + exit 2 +elif ! grep "%[3]s" /proc/1/attr/current; then + echo "FAILURE: not running with expected profile %[3]s" + exit 3 +fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name) + + if !runOnce { + testCmd = fmt.Sprintf(`while true; do +%s +sleep 1 +done`, testCmd) + } + + pod := &api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-apparmor-", + Annotations: map[string]string{ + apparmor.ContainerAnnotationKeyPrefix + "test": profile, + }, + Labels: map[string]string{ + "test": "apparmor", + }, + }, + Spec: api.PodSpec{ + Containers: []api.Container{{ + Name: "test", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"sh", "-c", testCmd}, + }}, + RestartPolicy: api.RestartPolicyNever, + }, + } + + if runOnce { + pod = f.PodClient().Create(pod) + framework.ExpectNoError(framework.WaitForPodSuccessInNamespace( + f.ClientSet, pod.Name, f.Namespace.Name)) + } else { + pod = f.PodClient().CreateSync(pod) + framework.ExpectNoError(f.WaitForPodReady(pod.Name)) + } + + return pod +} + +func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) { + profileName := appArmorProfilePrefix + f.Namespace.Name + profile := fmt.Sprintf(`#include +profile %s flags=(attach_disconnected) { + #include + + file, + + deny %s w, + audit %s w, +} +`, profileName, appArmorDeniedPath, appArmorAllowedPath) + + cm := &api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apparmor-profiles", + Namespace: f.Namespace.Name, + }, + Data: map[string]string{ + profileName: profile, + }, + } + return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) +} + +func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) { + True := true + // Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml + loader := &extensions.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apparmor-loader", + Namespace: f.Namespace.Name, + }, + Spec: extensions.DaemonSetSpec{ + Template: api.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"name": "apparmor-loader"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{{ + Name: "apparmor-loader", + Image: "gcr.io/google_containers/apparmor-loader:0.1", + Args: []string{"-poll", "10s", "/profiles"}, + SecurityContext: &api.SecurityContext{ + Privileged: &True, + }, + VolumeMounts: []api.VolumeMount{{ + Name: "sys", + MountPath: "/sys", + ReadOnly: true, + }, { + Name: "apparmor-includes", + MountPath: "/etc/apparmor.d", + ReadOnly: true, + }, { + Name: "profiles", + MountPath: "/profiles", + ReadOnly: true, + }}, + }}, + Volumes: []api.Volume{{ + Name: "sys", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: "/sys", + }, + }, + }, { + Name: "apparmor-includes", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: "/etc/apparmor.d", + }, + }, + }, { + Name: "profiles", + VolumeSource: api.VolumeSource{ + ConfigMap: &api.ConfigMapVolumeSource{ + LocalObjectReference: api.LocalObjectReference{ + Name: "apparmor-profiles", + }, + }, + }, + }}, + }, + }, + }, + } + return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader) +} diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index 742b3508345..a7fa15e8375 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -10,6 +10,7 @@ load( go_library( name = "go_default_library", srcs = [ + "apparmor.go", "configmaps.go", "daemonsets.go", "deployments.go", @@ -36,6 +37,7 @@ go_library( "//test/e2e/framework:go_default_library", "//vendor:github.com/onsi/ginkgo", "//vendor:github.com/onsi/gomega", + "//vendor:github.com/onsi/gomega/gstruct", "//vendor:k8s.io/apimachinery/pkg/api/errors", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/labels", diff --git a/test/e2e/upgrades/apparmor.go b/test/e2e/upgrades/apparmor.go new file mode 100644 index 00000000000..44cf56aba71 --- /dev/null +++ b/test/e2e/upgrades/apparmor.go @@ -0,0 +1,99 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrades + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + api "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/test/e2e/common" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" +) + +// AppArmorUpgradeTest tests that AppArmor profiles are enforced & usable across upgrades. +type AppArmorUpgradeTest struct { + pod *api.Pod +} + +func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" } + +// Setup creates a secret and then verifies that a pod can consume it. +func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) { + common.SkipIfAppArmorNotSupported() + By("Loading AppArmor profiles to nodes") + common.LoadAppArmorProfiles(f) + + // Create the initial test pod. + By("Creating a long-running AppArmor enabled pod.") + t.pod = common.CreateAppArmorTestPod(f, false) + + // Verify initial state. + t.verifyNodesAppArmorEnabled(f) + t.verifyNewPodSucceeds(f) +} + +// Test waits for the upgrade to complete, and then verifies that a +// pod can still consume the secret. +func (t *AppArmorUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) { + <-done + if upgrade == MasterUpgrade { + t.verifyPodStillUp(f) + } + t.verifyNodesAppArmorEnabled(f) + t.verifyNewPodSucceeds(f) +} + +// Teardown cleans up any remaining resources. +func (t *AppArmorUpgradeTest) Teardown(f *framework.Framework) { + // rely on the namespace deletion to clean up everything + By("Logging container failures") + framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) +} + +func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) { + By("Verifying an AppArmor profile is continuously enforced for a pod") + pod, err := f.PodClient().Get(t.pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Should be able to get pod") + Expect(pod.Status.Phase).To(Equal(api.PodRunning), "Pod should stay running") + Expect(pod.Status.ContainerStatuses[0].State.Running).NotTo(BeNil(), "Container should be running") + Expect(pod.Status.ContainerStatuses[0].RestartCount).To(BeZero(), "Container should not need to be restarted") +} + +func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) { + By("Verifying an AppArmor profile is enforced for a new pod") + common.CreateAppArmorTestPod(f, true) +} + +func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) { + By("Verifying nodes are AppArmor enabled") + nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{}) + framework.ExpectNoError(err, "Failed to list nodes") + for _, node := range nodes.Items { + Expect(node.Status.Conditions).To(gstruct.MatchElements(conditionType, gstruct.IgnoreExtras, gstruct.Elements{ + "Ready": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Message": ContainSubstring("AppArmor enabled"), + }), + })) + } +} + +func conditionType(condition interface{}) string { + return string(condition.(api.NodeCondition).Type) +}