github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 04:33:26 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #84439 from danielqsj/f-apparmor

Browse Source 
refactor apparmor utils in e2e
This commit is contained in:
Kubernetes Prow Robot 2019-11-14 13:30:08 -08:00 committed by GitHub
commit 55cef8048c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 104 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/auth/BUILD
View File

@ -51,7 +51,6 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/auth:go_default_library",
"//test/e2e/framework/deployment:go_default_library",

3
test/e2e/auth/pod_security_policy.go
View File

@ -31,7 +31,6 @@ import (
"k8s.io/kubernetes/pkg/security/apparmor"
"k8s.io/kubernetes/pkg/security/podsecuritypolicy/seccomp"
psputil "k8s.io/kubernetes/pkg/security/podsecuritypolicy/util"
"k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/auth"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -168,7 +167,7 @@ func testPrivilegedPods(tester func(pod *v1.Pod)) {
tester(hostipc)
})
if common.IsAppArmorSupported() {
if framework.IsAppArmorSupported() {
ginkgo.By("Running a custom AppArmor profile pod", func() {
aa := restrictedPod("apparmor")
// Every node is expected to have the docker-default profile.

2
test/e2e/common/BUILD
View File

@ -8,7 +8,6 @@ load(
go_library(
name = "go_default_library",
srcs = [
"apparmor.go",
"configmap.go",
"configmap_volume.go",
"container.go",
@ -51,7 +50,6 @@ go_library(
"//pkg/kubelet/images:go_default_library",
"//pkg/kubelet/runtimeclass/testing:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//pkg/security/apparmor:go_default_library",
"//staging/src/k8s.io/api/coordination/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",

2
test/e2e/framework/BUILD
View File

@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"checks.go",
"cleanup.go",
"exec_util.go",
"expect.go",
@ -133,6 +134,7 @@ filegroup(
"//test/e2e/framework/pv:all-srcs",
"//test/e2e/framework/replicaset:all-srcs",
"//test/e2e/framework/resource:all-srcs",
"//test/e2e/framework/security:all-srcs",
"//test/e2e/framework/service:all-srcs",
"//test/e2e/framework/ssh:all-srcs",
"//test/e2e/framework/statefulset:all-srcs",

22
test/e2e/framework/checks.go Normal file
View File

@ -0,0 +1,22 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
// IsAppArmorSupported checks whether the AppArmor is supported by the node OS distro.
func IsAppArmorSupported() bool {
return NodeOSDistroIs(AppArmorDistros...)
}

32
test/e2e/framework/security/BUILD Normal file
View File

@ -0,0 +1,32 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["apparmor.go"],
importpath = "k8s.io/kubernetes/test/e2e/framework/security",
visibility = ["//visibility:public"],
deps = [
"//pkg/security/apparmor:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/pod:go_default_library",
"//test/utils/image:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

63
test/e2e/common/apparmor.go → test/e2e/framework/security/apparmor.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package common
package security
import (
"fmt"
@ -22,6 +22,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/security/apparmor"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -37,27 +38,17 @@ const (
loaderLabelValue = "e2e-apparmor-loader"
)
// AppArmorDistros are distros with AppArmor support
var AppArmorDistros = []string{"gci", "ubuntu"}
func IsAppArmorSupported() bool {
return framework.NodeOSDistroIs(AppArmorDistros...)
}
func SkipIfAppArmorNotSupported() {
framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...)
}
func LoadAppArmorProfiles(f *framework.Framework) {
createAppArmorProfileCM(f)
createAppArmorProfileLoader(f)
// LoadAppArmorProfiles creates apparmor-profiles ConfigMap and apparmor-loader ReplicationController.
func LoadAppArmorProfiles(nsName string, clientset clientset.Interface) {
createAppArmorProfileCM(nsName, clientset)
createAppArmorProfileLoader(nsName, clientset)
}
// CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with
// an error code if the profile is incorrectly enforced. If runOnce is true the pod will exit after
// a single test, otherwise it will repeat the test every 1 second until failure.
func CreateAppArmorTestPod(f *framework.Framework, unconfined bool, runOnce bool) *v1.Pod {
profile := "localhost/" + appArmorProfilePrefix + f.Namespace.Name
func CreateAppArmorTestPod(nsName string, clientset clientset.Interface, podClient *framework.PodClient, unconfined bool, runOnce bool) *v1.Pod {
profile := "localhost/" + appArmorProfilePrefix + nsName
testCmd := fmt.Sprintf(`
if touch %[1]s; then
echo "FAILURE: write to %[1]s should be denied"
@ -69,7 +60,7 @@ elif [[ $(< /proc/self/attr/current) != "%[3]s" ]]; then
echo "FAILURE: not running with expected profile %[3]s"
echo "found: $(cat /proc/self/attr/current)"
exit 3
fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+f.Namespace.Name)
fi`, appArmorDeniedPath, appArmorAllowedPath, appArmorProfilePrefix+nsName)
if unconfined {
profile = apparmor.ProfileNameUnconfined
@ -93,7 +84,7 @@ done`, testCmd)
loaderAffinity := &v1.Affinity{
PodAffinity: &v1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{
Namespaces: []string{f.Namespace.Name},
Namespaces: []string{nsName},
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{loaderLabelKey: loaderLabelValue},
},
@ -124,26 +115,26 @@ done`, testCmd)
}
if runOnce {
pod = f.PodClient().Create(pod)
pod = podClient.Create(pod)
framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(
f.ClientSet, pod.Name, f.Namespace.Name))
clientset, pod.Name, nsName))
var err error
pod, err = f.PodClient().Get(pod.Name, metav1.GetOptions{})
pod, err = podClient.Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
} else {
pod = f.PodClient().CreateSync(pod)
framework.ExpectNoError(f.WaitForPodReady(pod.Name))
pod = podClient.CreateSync(pod)
framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(clientset, pod.Name, nsName, framework.PodStartTimeout))
}
// Verify Pod affinity colocated the Pods.
loader := getRunningLoaderPod(f)
loader := getRunningLoaderPod(nsName, clientset)
framework.ExpectEqual(pod.Spec.NodeName, loader.Spec.NodeName)
return pod
}
func createAppArmorProfileCM(f *framework.Framework) {
profileName := appArmorProfilePrefix + f.Namespace.Name
func createAppArmorProfileCM(nsName string, clientset clientset.Interface) {
profileName := appArmorProfilePrefix + nsName
profile := fmt.Sprintf(`#include <tunables/global>
profile %s flags=(attach_disconnected) {
#include <abstractions/base>
@ -158,23 +149,23 @@ profile %s flags=(attach_disconnected) {
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "apparmor-profiles",
Namespace: f.Namespace.Name,
Namespace: nsName,
},
Data: map[string]string{
profileName: profile,
},
}
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(cm)
_, err := clientset.CoreV1().ConfigMaps(nsName).Create(cm)
framework.ExpectNoError(err, "Failed to create apparmor-profiles ConfigMap")
}
func createAppArmorProfileLoader(f *framework.Framework) {
func createAppArmorProfileLoader(nsName string, clientset clientset.Interface) {
True := true
One := int32(1)
loader := &v1.ReplicationController{
ObjectMeta: metav1.ObjectMeta{
Name: "apparmor-loader",
Namespace: f.Namespace.Name,
Namespace: nsName,
},
Spec: v1.ReplicationControllerSpec{
Replicas: &One,
@ -232,18 +223,18 @@ func createAppArmorProfileLoader(f *framework.Framework) {
},
},
}
_, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(loader)
_, err := clientset.CoreV1().ReplicationControllers(nsName).Create(loader)
framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController")
// Wait for loader to be ready.
getRunningLoaderPod(f)
getRunningLoaderPod(nsName, clientset)
}
func getRunningLoaderPod(f *framework.Framework) *v1.Pod {
func getRunningLoaderPod(nsName string, clientset clientset.Interface) *v1.Pod {
label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue}))
pods, err := e2epod.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label)
pods, err := e2epod.WaitForPodsWithLabelScheduled(clientset, nsName, label)
framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod")
pod := &pods.Items[0]
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(f.ClientSet, pod), "Failed to run apparmor-loader Pod")
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(clientset, pod), "Failed to run apparmor-loader Pod")
return pod
}

8
test/e2e/framework/skip.go
View File

@ -174,3 +174,11 @@ func serverVersionGTE(v *utilversion.Version, c discovery.ServerVersionInterface
}
return sv.AtLeast(v), nil
}
// AppArmorDistros are distros with AppArmor support
var AppArmorDistros = []string{"gci", "ubuntu"}
// SkipIfAppArmorNotSupported skips if the AppArmor is not supported by the node OS distro.
func SkipIfAppArmorNotSupported() {
SkipUnlessNodeOSDistroIs(AppArmorDistros...)
}

2
test/e2e/node/BUILD
View File

@ -38,13 +38,13 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/job:go_default_library",
"//test/e2e/framework/kubelet:go_default_library",
"//test/e2e/framework/node:go_default_library",
"//test/e2e/framework/perf:go_default_library",
"//test/e2e/framework/pod:go_default_library",
"//test/e2e/framework/security:go_default_library",
"//test/e2e/framework/ssh:go_default_library",
"//test/e2e/framework/volume:go_default_library",
"//test/e2e/perftype:go_default_library",

10
test/e2e/node/apparmor.go
View File

@ -17,8 +17,8 @@ limitations under the License.
package node
import (
"k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
e2esecurity "k8s.io/kubernetes/test/e2e/framework/security"
"github.com/onsi/ginkgo"
)
@ -28,8 +28,8 @@ var _ = SIGDescribe("AppArmor", func() {
ginkgo.Context("load AppArmor profiles", func() {
ginkgo.BeforeEach(func() {
common.SkipIfAppArmorNotSupported()
common.LoadAppArmorProfiles(f)
framework.SkipIfAppArmorNotSupported()
e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet)
})
ginkgo.AfterEach(func() {
if !ginkgo.CurrentGinkgoTestDescription().Failed {
@ -39,11 +39,11 @@ var _ = SIGDescribe("AppArmor", func() {
})
ginkgo.It("should enforce an AppArmor profile", func() {
common.CreateAppArmorTestPod(f, false, true)
e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true)
})
ginkgo.It("can disable an AppArmor profile, using unconfined", func() {
common.CreateAppArmorTestPod(f, true, true)
e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), true, true)
})
})
})

2
test/e2e/upgrades/BUILD
View File

@ -34,11 +34,11 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/autoscaling:go_default_library",
"//test/e2e/framework/job:go_default_library",
"//test/e2e/framework/node:go_default_library",
"//test/e2e/framework/security:go_default_library",
"//test/e2e/framework/service:go_default_library",
"//test/e2e/framework/statefulset:go_default_library",
"//test/e2e/framework/testfiles:go_default_library",

10
test/e2e/upgrades/apparmor.go
View File

@ -19,8 +19,8 @@ package upgrades
import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
e2esecurity "k8s.io/kubernetes/test/e2e/framework/security"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@ -38,7 +38,7 @@ func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
// Skip returns true when this test can be skipped.
func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
supportedImages := make(map[string]bool)
for _, d := range common.AppArmorDistros {
for _, d := range framework.AppArmorDistros {
supportedImages[d] = true
}
@ -53,11 +53,11 @@ func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
// Setup creates a secret and then verifies that a pod can consume it.
func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
ginkgo.By("Loading AppArmor profiles to nodes")
common.LoadAppArmorProfiles(f)
e2esecurity.LoadAppArmorProfiles(f.Namespace.Name, f.ClientSet)
// Create the initial test pod.
ginkgo.By("Creating a long-running AppArmor enabled pod.")
t.pod = common.CreateAppArmorTestPod(f, false, false)
t.pod = e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, false)
// Verify initial state.
t.verifyNodesAppArmorEnabled(f)
@ -93,7 +93,7 @@ func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
ginkgo.By("Verifying an AppArmor profile is enforced for a new pod")
common.CreateAppArmorTestPod(f, false, true)
e2esecurity.CreateAppArmorTestPod(f.Namespace.Name, f.ClientSet, f.PodClient(), false, true)
}
func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {