github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 03:41:45 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #70036 from pbarker/audit-etoe

Browse Source 
dynamic audit e2e test
This commit is contained in:
Kubernetes Prow Robot 2019-03-06 17:58:58 -08:00 committed by GitHub
commit ab7a48d796
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 396 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cluster/gce/config-test.sh
View File

@ -376,6 +376,8 @@ else
ADMISSION_CONTROL=${KUBE_ADMISSION_CONTROL}
fi
ENABLE_APISERVER_DYNAMIC_AUDIT="${ENABLE_APISERVER_DYNAMIC_AUDIT:-false}"
# Optional: if set to true kube-up will automatically check for existing resources and clean them up.
KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

5
cluster/gce/gci/configure-helper.sh
View File

@ -1709,6 +1709,11 @@ function start-kube-apiserver {
fi
fi
if [[ "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}" == "true" ]]; then
params+=" --audit-dynamic-configuration"
RUNTIME_CONFIG="${RUNTIME_CONFIG},auditconfiguration.k8s.io/v1alpha1=true"
fi
if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then
params+=" --enable-logs-handler=false"
fi

1
cluster/gce/util.sh
View File

@ -1111,6 +1111,7 @@ MULTIZONE: $(yaml-quote ${MULTIZONE:-})
NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-})
ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-})
ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-})
ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_DYNAMIC_AUDIT:-})
ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false})
ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote ${ENABLE_PATCH_CONVERSION_DETECTOR:-false})
ADVANCED_AUDIT_POLICY: $(yaml-quote ${ADVANCED_AUDIT_POLICY:-})

3
test/e2e/auth/BUILD
View File

@ -9,6 +9,7 @@ go_library(
name = "go_default_library",
srcs = [
"audit.go",
"audit_dynamic.go",
"certificates.go",
"framework.go",
"metadata_concealment.go",
@ -26,6 +27,7 @@ go_library(
"//pkg/security/podsecuritypolicy/util:go_default_library",
"//plugin/pkg/admission/serviceaccount:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/batch/v1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
@ -39,6 +41,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

381
test/e2e/auth/audit_dynamic.go Normal file
View File

@ -0,0 +1,381 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo"
auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
)
var _ = SIGDescribe("[Feature:DynamicAudit]", func() {
f := framework.NewDefaultFramework("audit")
It("should dynamically audit API calls", func() {
namespace := f.Namespace.Name
By("Creating a kubernetes client that impersonates an unauthorized anonymous user")
config, err := framework.LoadConfig()
framework.ExpectNoError(err, "failed to fetch config")
config.Impersonate = restclient.ImpersonationConfig{
UserName: "system:anonymous",
Groups: []string{"system:unauthenticated"},
}
anonymousClient, err := clientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create the anonymous client")
_, err = f.ClientSet.CoreV1().Namespaces().Create(&apiv1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "audit",
},
})
framework.ExpectNoError(err, "failed to create namespace")
_, err = f.ClientSet.CoreV1().Pods(namespace).Create(&apiv1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "audit-proxy",
Labels: map[string]string{
"app": "audit",
},
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
{
Name: "proxy",
Image: imageutils.GetE2EImage(imageutils.AuditProxy),
Ports: []apiv1.ContainerPort{
{
ContainerPort: 8080,
},
},
},
},
},
})
framework.ExpectNoError(err, "failed to create proxy pod")
_, err = f.ClientSet.CoreV1().Services(namespace).Create(&apiv1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "audit",
},
Spec: apiv1.ServiceSpec{
Ports: []apiv1.ServicePort{
{
Port: 80,
TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
},
},
Selector: map[string]string{
"app": "audit",
},
},
})
framework.ExpectNoError(err, "failed to create proxy service")
config, err = framework.LoadConfig()
framework.ExpectNoError(err, "failed to load config")
var podIP string
// get pod ip
err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
p, err := f.ClientSet.CoreV1().Pods(namespace).Get("audit-proxy", metav1.GetOptions{})
if errors.IsNotFound(err) {
framework.Logf("waiting for audit-proxy pod to be present")
return false, nil
} else if err != nil {
return false, err
}
podIP = p.Status.PodIP
if podIP == "" {
framework.Logf("waiting for audit-proxy pod IP to be ready")
return false, nil
}
return true, nil
})
framework.ExpectNoError(err, "timed out waiting for audit-proxy pod to be ready")
podURL := fmt.Sprintf("http://%s:8080", podIP)
// create audit sink
sink := auditregv1alpha1.AuditSink{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: auditregv1alpha1.AuditSinkSpec{
Policy: auditregv1alpha1.Policy{
Level: auditregv1alpha1.LevelRequestResponse,
Stages: []auditregv1alpha1.Stage{
auditregv1alpha1.StageRequestReceived,
auditregv1alpha1.StageResponseStarted,
auditregv1alpha1.StageResponseComplete,
auditregv1alpha1.StagePanic,
},
},
Webhook: auditregv1alpha1.Webhook{
ClientConfig: auditregv1alpha1.WebhookClientConfig{
URL: &podURL,
},
},
},
}
_, err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Create(&sink)
framework.ExpectNoError(err, "failed to create audit sink")
framework.Logf("created audit sink")
// check that we are receiving logs in the proxy
err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
if err != nil {
framework.Logf("waiting for audit-proxy pod logs to be available")
return false, nil
}
if logs == "" {
framework.Logf("waiting for audit-proxy pod logs to be non-empty")
return false, nil
}
return true, nil
})
framework.ExpectNoError(err, "failed to get logs from audit-proxy pod")
auditTestUser = "kubernetes-admin"
testCases := []struct {
action func()
events []utils.AuditEvent
}{
// Create, get, update, patch, delete, list, watch pods.
// TODO(@pbarker): dedupe this with the main audit test once policy functionality is available
// https://github.com/kubernetes/kubernetes/issues/70818
{
func() {
pod := &apiv1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "audit-pod",
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{{
Name: "pause",
Image: imageutils.GetPauseImageName(),
}},
},
}
updatePod := func(pod *apiv1.Pod) {}
f.PodClient().CreateSync(pod)
_, err := f.PodClient().Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get audit-pod")
podChan, err := f.PodClient().Watch(watchOptions)
framework.ExpectNoError(err, "failed to create watch for pods")
for range podChan.ResultChan() {
}
f.PodClient().Update(pod.Name, updatePod)
_, err = f.PodClient().List(metav1.ListOptions{})
framework.ExpectNoError(err, "failed to list pods")
_, err = f.PodClient().Patch(pod.Name, types.JSONPatchType, patch)
framework.ExpectNoError(err, "failed to patch pod")
f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
},
[]utils.AuditEvent{
{
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
Verb: "create",
Code: 201,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "get",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
Verb: "list",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseStarted,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
Verb: "watch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: false,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
Verb: "watch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: false,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "update",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "patch",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
}, {
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
Verb: "delete",
Code: 200,
User: auditTestUser,
Resource: "pods",
Namespace: namespace,
RequestObject: true,
ResponseObject: true,
AuthorizeDecision: "allow",
},
},
},
}
// test authorizer annotations, RBAC is required.
annotationTestCases := []struct {
action func()
events []utils.AuditEvent
}{
// get a pod with unauthorized user
{
func() {
_, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{})
expectForbidden(err)
},
[]utils.AuditEvent{
{
Level: auditinternal.LevelRequestResponse,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace),
Verb: "get",
Code: 403,
User: auditTestUser,
ImpersonatedUser: "system:anonymous",
ImpersonatedGroups: "system:unauthenticated",
Resource: "pods",
Namespace: namespace,
RequestObject: false,
ResponseObject: true,
AuthorizeDecision: "forbid",
},
},
},
}
if framework.IsRBACEnabled(f) {
testCases = append(testCases, annotationTestCases...)
}
expectedEvents := []utils.AuditEvent{}
for _, t := range testCases {
t.action()
expectedEvents = append(expectedEvents, t.events...)
}
// The default flush timeout is 30 seconds, therefore it should be enough to retry once
// to find all expected events. However, we're waiting for 5 minutes to avoid flakes.
pollingInterval := 30 * time.Second
pollingTimeout := 5 * time.Minute
err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
// Fetch the logs
logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
if err != nil {
return false, err
}
reader := strings.NewReader(logs)
missingReport, err := utils.CheckAuditLines(reader, expectedEvents, auditv1.SchemeGroupVersion)
if err != nil {
framework.Logf("Failed to observe audit events: %v", err)
} else if len(missingReport.MissingEvents) > 0 {
framework.Logf(missingReport.String())
}
return len(missingReport.MissingEvents) == 0, nil
})
framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout)
err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Delete("test", &metav1.DeleteOptions{})
framework.ExpectNoError(err, "could not delete audit configuration")
})
})

1
test/e2e/common/util.go
View File

@ -48,6 +48,7 @@ var CurrentSuite Suite
// only used by node e2e test.
// TODO(random-liu): Change the image puller pod to use similar mechanism.
var CommonImageWhiteList = sets.NewString(
imageutils.GetE2EImage(imageutils.AuditProxy),
imageutils.GetE2EImage(imageutils.BusyBox),
imageutils.GetE2EImage(imageutils.EntrypointTester),
imageutils.GetE2EImage(imageutils.IpcUtils),

3
test/utils/image/manifest.go
View File

@ -105,6 +105,8 @@ const (
APIServer
// AppArmorLoader image
AppArmorLoader
// AuditProxy image
AuditProxy
// BusyBox image
BusyBox
// CheckMetadataConcealment image
@ -196,6 +198,7 @@ func initImageConfigs() map[int]Config {
configs[AdmissionWebhook] = Config{e2eRegistry, "webhook", "1.14v1"}
configs[APIServer] = Config{e2eRegistry, "sample-apiserver", "1.10"}
configs[AppArmorLoader] = Config{e2eRegistry, "apparmor-loader", "1.0"}
configs[AuditProxy] = Config{e2eRegistry, "audit-proxy", "1.0"}
configs[BusyBox] = Config{dockerLibraryRegistry, "busybox", "1.29"}
configs[CheckMetadataConcealment] = Config{e2eRegistry, "metadata-concealment", "1.2"}
configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"}