mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-11-03 23:40:03 +00:00
Merge pull request #42349 from timstclair/aa-upgrade
Automatic merge from submit-queue (batch tested with PRs 41794, 42349, 42755, 42901, 42933) AppArmor cluster upgrade test Add a cluster upgrade test for AppArmor. I still need to test this (having some trouble with the cluster-upgrade tests), but wanted to start the review process. /cc @dchen1107 @roberthbailey
This commit is contained in:
Kubernetes Submit Queue
GitHub
@@ -144,7 +144,6 @@ go_library(
|
||||
"//pkg/master/ports:go_default_library",
|
||||
"//pkg/metrics:go_default_library",
|
||||
"//pkg/quota/evaluator/core:go_default_library",
|
||||
"//pkg/security/apparmor:go_default_library",
|
||||
"//pkg/util:go_default_library",
|
||||
"//pkg/util/exec:go_default_library",
|
||||
"//pkg/util/logs:go_default_library",
|
||||
|
||||
@@ -17,162 +17,22 @@ limitations under the License.
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
api "k8s.io/kubernetes/pkg/api/v1"
|
||||
extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
|
||||
"k8s.io/kubernetes/pkg/security/apparmor"
|
||||
"k8s.io/kubernetes/test/e2e/common"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
)
|
||||
|
||||
const (
|
||||
profilePrefix = "e2e-apparmor-test-"
|
||||
allowedPath = "/expect_allowed_write"
|
||||
deniedPath = "/expect_permission_denied"
|
||||
)
|
||||
|
||||
var _ = framework.KubeDescribe("AppArmor", func() {
|
||||
f := framework.NewDefaultFramework("apparmor")
|
||||
|
||||
BeforeEach(func() {
|
||||
SkipIfAppArmorNotSupported()
|
||||
LoadAppArmorProfiles(f)
|
||||
common.SkipIfAppArmorNotSupported()
|
||||
common.LoadAppArmorProfiles(f)
|
||||
})
|
||||
|
||||
It("should enforce an AppArmor profile", func() {
|
||||
profile := "localhost/" + profilePrefix + f.Namespace.Name
|
||||
testCmd := fmt.Sprintf(`
|
||||
if touch %[1]s; then
|
||||
echo "FAILURE: write to %[1]s should be denied"
|
||||
exit 1
|
||||
elif ! touch %[2]s; then
|
||||
echo "FAILURE: write to %[2]s should be allowed"
|
||||
exit 2
|
||||
fi`, deniedPath, allowedPath)
|
||||
pod := &api.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-apparmor",
|
||||
Annotations: map[string]string{
|
||||
apparmor.ContainerAnnotationKeyPrefix + "test": profile,
|
||||
},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{{
|
||||
Name: "test",
|
||||
Image: "gcr.io/google_containers/busybox:1.24",
|
||||
Command: []string{"sh", "-c", testCmd},
|
||||
}},
|
||||
RestartPolicy: api.RestartPolicyNever,
|
||||
},
|
||||
}
|
||||
f.PodClient().Create(pod)
|
||||
framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(
|
||||
f.ClientSet, pod.Name, f.Namespace.Name))
|
||||
common.CreateAppArmorTestPod(f, true)
|
||||
framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
|
||||
})
|
||||
})
|
||||
|
||||
func SkipIfAppArmorNotSupported() {
|
||||
framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
|
||||
}
|
||||
|
||||
func LoadAppArmorProfiles(f *framework.Framework) {
|
||||
_, err := createAppArmorProfileCM(f)
|
||||
framework.ExpectNoError(err)
|
||||
_, err = createAppArmorProfileLoader(f)
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
|
||||
func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) {
|
||||
profileName := profilePrefix + f.Namespace.Name
|
||||
profile := fmt.Sprintf(`#include <tunables/global>
|
||||
profile %s flags=(attach_disconnected) {
|
||||
#include <abstractions/base>
|
||||
|
||||
file,
|
||||
|
||||
deny %s w,
|
||||
audit %s w,
|
||||
}
|
||||
`, profileName, deniedPath, allowedPath)
|
||||
|
||||
cm := &api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "apparmor-profiles",
|
||||
Namespace: f.Namespace.Name,
|
||||
},
|
||||
Data: map[string]string{
|
||||
profileName: profile,
|
||||
},
|
||||
}
|
||||
return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm)
|
||||
}
|
||||
|
||||
func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) {
|
||||
True := true
|
||||
// Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml
|
||||
loader := &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "apparmor-loader",
|
||||
Namespace: f.Namespace.Name,
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: api.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{"name": "apparmor-loader"},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{{
|
||||
Name: "apparmor-loader",
|
||||
Image: "gcr.io/google_containers/apparmor-loader:0.1",
|
||||
Args: []string{"-poll", "10s", "/profiles"},
|
||||
SecurityContext: &api.SecurityContext{
|
||||
Privileged: &True,
|
||||
},
|
||||
VolumeMounts: []api.VolumeMount{{
|
||||
Name: "sys",
|
||||
MountPath: "/sys",
|
||||
ReadOnly: true,
|
||||
}, {
|
||||
Name: "apparmor-includes",
|
||||
MountPath: "/etc/apparmor.d",
|
||||
ReadOnly: true,
|
||||
}, {
|
||||
Name: "profiles",
|
||||
MountPath: "/profiles",
|
||||
ReadOnly: true,
|
||||
}},
|
||||
}},
|
||||
Volumes: []api.Volume{{
|
||||
Name: "sys",
|
||||
VolumeSource: api.VolumeSource{
|
||||
HostPath: &api.HostPathVolumeSource{
|
||||
Path: "/sys",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "apparmor-includes",
|
||||
VolumeSource: api.VolumeSource{
|
||||
HostPath: &api.HostPathVolumeSource{
|
||||
Path: "/etc/apparmor.d",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "profiles",
|
||||
VolumeSource: api.VolumeSource{
|
||||
ConfigMap: &api.ConfigMapVolumeSource{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "apparmor-profiles",
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader)
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ var upgradeTests = []upgrades.Test{
|
||||
&upgrades.PersistentVolumeUpgradeTest{},
|
||||
&upgrades.DaemonSetUpgradeTest{},
|
||||
&upgrades.IngressUpgradeTest{},
|
||||
&upgrades.AppArmorUpgradeTest{},
|
||||
}
|
||||
|
||||
var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
|
||||
|
||||
@@ -10,6 +10,7 @@ load(
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"apparmor.go",
|
||||
"autoscaling_utils.go",
|
||||
"configmap.go",
|
||||
"container_probe.go",
|
||||
@@ -36,11 +37,13 @@ go_library(
|
||||
"//pkg/api/v1:go_default_library",
|
||||
"//pkg/api/v1/pod:go_default_library",
|
||||
"//pkg/apis/autoscaling/v1:go_default_library",
|
||||
"//pkg/apis/extensions/v1beta1:go_default_library",
|
||||
"//pkg/client/clientset_generated/clientset:go_default_library",
|
||||
"//pkg/client/clientset_generated/internalclientset:go_default_library",
|
||||
"//pkg/client/conditions:go_default_library",
|
||||
"//pkg/kubelet:go_default_library",
|
||||
"//pkg/kubelet/sysctl:go_default_library",
|
||||
"//pkg/security/apparmor:go_default_library",
|
||||
"//test/e2e/framework:go_default_library",
|
||||
"//test/utils:go_default_library",
|
||||
"//vendor:github.com/golang/glog",
|
||||
|
||||
191
test/e2e/common/apparmor.go
Normal file
191
test/e2e/common/apparmor.go
Normal file
@@ -0,0 +1,191 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
api "k8s.io/kubernetes/pkg/api/v1"
|
||||
extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
|
||||
"k8s.io/kubernetes/pkg/security/apparmor"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
)
|
||||
|
||||
const (
|
||||
appArmorProfilePrefix = "e2e-apparmor-test-"
|
||||
appArmorAllowedPath = "/expect_allowed_write"
|
||||
appArmorDeniedPath = "/expect_permission_denied"
|
||||
)
|
||||
|
||||
func SkipIfAppArmorNotSupported() {
|
||||
framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
|
||||
}
|
||||
|
||||
func LoadAppArmorProfiles(f *framework.Framework) {
|
||||
_, err := createAppArmorProfileCM(f)
|
||||
framework.ExpectNoError(err)
|
||||
_, err = createAppArmorProfileLoader(f)
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
|
||||
// CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with
|
||||
// an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after
|
||||
// a single test, otherwise it will repeat the test every 1 second until failure.
|
||||
func CreateAppArmorTestPod(f *framework.Framework, runOnce bool) *api.Pod {
|
||||
profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name
|
||||
testCmd := fmt.Sprintf(`
|
||||
if touch %[1]s; then
|
||||
echo "FAILURE: write to %[1]s should be denied"
|
||||
exit 1
|
||||
elif ! touch %[2]s; then
|
||||
echo "FAILURE: write to %[2]s should be allowed"
|
||||
exit 2
|
||||
elif ! grep "%[3]s" /proc/1/attr/current; then
|
||||
echo "FAILURE: not running with expected profile %[3]s"
|
||||
exit 3
|
||||
fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name)
|
||||
|
||||
if !runOnce {
|
||||
testCmd = fmt.Sprintf(`while true; do
|
||||
%s
|
||||
sleep 1
|
||||
done`, testCmd)
|
||||
}
|
||||
|
||||
pod := &api.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "test-apparmor-",
|
||||
Annotations: map[string]string{
|
||||
apparmor.ContainerAnnotationKeyPrefix + "test": profile,
|
||||
},
|
||||
Labels: map[string]string{
|
||||
"test": "apparmor",
|
||||
},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{{
|
||||
Name: "test",
|
||||
Image: "gcr.io/google_containers/busybox:1.24",
|
||||
Command: []string{"sh", "-c", testCmd},
|
||||
}},
|
||||
RestartPolicy: api.RestartPolicyNever,
|
||||
},
|
||||
}
|
||||
|
||||
if runOnce {
|
||||
pod = f.PodClient().Create(pod)
|
||||
framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(
|
||||
f.ClientSet, pod.Name, f.Namespace.Name))
|
||||
} else {
|
||||
pod = f.PodClient().CreateSync(pod)
|
||||
framework.ExpectNoError(f.WaitForPodReady(pod.Name))
|
||||
}
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) {
|
||||
profileName := appArmorProfilePrefix + f.Namespace.Name
|
||||
profile := fmt.Sprintf(`#include <tunables/global>
|
||||
profile %s flags=(attach_disconnected) {
|
||||
#include <abstractions/base>
|
||||
|
||||
file,
|
||||
|
||||
deny %s w,
|
||||
audit %s w,
|
||||
}
|
||||
`, profileName, appArmorDeniedPath, appArmorAllowedPath)
|
||||
|
||||
cm := &api.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "apparmor-profiles",
|
||||
Namespace: f.Namespace.Name,
|
||||
},
|
||||
Data: map[string]string{
|
||||
profileName: profile,
|
||||
},
|
||||
}
|
||||
return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm)
|
||||
}
|
||||
|
||||
func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) {
|
||||
True := true
|
||||
// Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml
|
||||
loader := &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "apparmor-loader",
|
||||
Namespace: f.Namespace.Name,
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: api.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{"name": "apparmor-loader"},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{{
|
||||
Name: "apparmor-loader",
|
||||
Image: "gcr.io/google_containers/apparmor-loader:0.1",
|
||||
Args: []string{"-poll", "10s", "/profiles"},
|
||||
SecurityContext: &api.SecurityContext{
|
||||
Privileged: &True,
|
||||
},
|
||||
VolumeMounts: []api.VolumeMount{{
|
||||
Name: "sys",
|
||||
MountPath: "/sys",
|
||||
ReadOnly: true,
|
||||
}, {
|
||||
Name: "apparmor-includes",
|
||||
MountPath: "/etc/apparmor.d",
|
||||
ReadOnly: true,
|
||||
}, {
|
||||
Name: "profiles",
|
||||
MountPath: "/profiles",
|
||||
ReadOnly: true,
|
||||
}},
|
||||
}},
|
||||
Volumes: []api.Volume{{
|
||||
Name: "sys",
|
||||
VolumeSource: api.VolumeSource{
|
||||
HostPath: &api.HostPathVolumeSource{
|
||||
Path: "/sys",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "apparmor-includes",
|
||||
VolumeSource: api.VolumeSource{
|
||||
HostPath: &api.HostPathVolumeSource{
|
||||
Path: "/etc/apparmor.d",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "profiles",
|
||||
VolumeSource: api.VolumeSource{
|
||||
ConfigMap: &api.ConfigMapVolumeSource{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "apparmor-profiles",
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader)
|
||||
}
|
||||
@@ -10,6 +10,7 @@ load(
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"apparmor.go",
|
||||
"configmaps.go",
|
||||
"daemonsets.go",
|
||||
"deployments.go",
|
||||
@@ -36,6 +37,7 @@ go_library(
|
||||
"//test/e2e/framework:go_default_library",
|
||||
"//vendor:github.com/onsi/ginkgo",
|
||||
"//vendor:github.com/onsi/gomega",
|
||||
"//vendor:github.com/onsi/gomega/gstruct",
|
||||
"//vendor:k8s.io/apimachinery/pkg/api/errors",
|
||||
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
|
||||
"//vendor:k8s.io/apimachinery/pkg/labels",
|
||||
|
||||
99
test/e2e/upgrades/apparmor.go
Normal file
99
test/e2e/upgrades/apparmor.go
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package upgrades
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
api "k8s.io/kubernetes/pkg/api/v1"
|
||||
"k8s.io/kubernetes/test/e2e/common"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/onsi/gomega/gstruct"
|
||||
)
|
||||
|
||||
// AppArmorUpgradeTest tests that AppArmor profiles are enforced & usable across upgrades.
|
||||
type AppArmorUpgradeTest struct {
|
||||
pod *api.Pod
|
||||
}
|
||||
|
||||
func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
|
||||
|
||||
// Setup creates a secret and then verifies that a pod can consume it.
|
||||
func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
|
||||
common.SkipIfAppArmorNotSupported()
|
||||
By("Loading AppArmor profiles to nodes")
|
||||
common.LoadAppArmorProfiles(f)
|
||||
|
||||
// Create the initial test pod.
|
||||
By("Creating a long-running AppArmor enabled pod.")
|
||||
t.pod = common.CreateAppArmorTestPod(f, false)
|
||||
|
||||
// Verify initial state.
|
||||
t.verifyNodesAppArmorEnabled(f)
|
||||
t.verifyNewPodSucceeds(f)
|
||||
}
|
||||
|
||||
// Test waits for the upgrade to complete, and then verifies that a
|
||||
// pod can still consume the secret.
|
||||
func (t *AppArmorUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
|
||||
<-done
|
||||
if upgrade == MasterUpgrade {
|
||||
t.verifyPodStillUp(f)
|
||||
}
|
||||
t.verifyNodesAppArmorEnabled(f)
|
||||
t.verifyNewPodSucceeds(f)
|
||||
}
|
||||
|
||||
// Teardown cleans up any remaining resources.
|
||||
func (t *AppArmorUpgradeTest) Teardown(f *framework.Framework) {
|
||||
// rely on the namespace deletion to clean up everything
|
||||
By("Logging container failures")
|
||||
framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
|
||||
}
|
||||
|
||||
func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
|
||||
By("Verifying an AppArmor profile is continuously enforced for a pod")
|
||||
pod, err := f.PodClient().Get(t.pod.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "Should be able to get pod")
|
||||
Expect(pod.Status.Phase).To(Equal(api.PodRunning), "Pod should stay running")
|
||||
Expect(pod.Status.ContainerStatuses[0].State.Running).NotTo(BeNil(), "Container should be running")
|
||||
Expect(pod.Status.ContainerStatuses[0].RestartCount).To(BeZero(), "Container should not need to be restarted")
|
||||
}
|
||||
|
||||
func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
|
||||
By("Verifying an AppArmor profile is enforced for a new pod")
|
||||
common.CreateAppArmorTestPod(f, true)
|
||||
}
|
||||
|
||||
func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {
|
||||
By("Verifying nodes are AppArmor enabled")
|
||||
nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
|
||||
framework.ExpectNoError(err, "Failed to list nodes")
|
||||
for _, node := range nodes.Items {
|
||||
Expect(node.Status.Conditions).To(gstruct.MatchElements(conditionType, gstruct.IgnoreExtras, gstruct.Elements{
|
||||
"Ready": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
||||
"Message": ContainSubstring("AppArmor enabled"),
|
||||
}),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
func conditionType(condition interface{}) string {
|
||||
return string(condition.(api.NodeCondition).Type)
|
||||
}
|
||||
Reference in New Issue
Block a user