github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-11-02 06:47:34 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #77824 from roycaihw/webhook-trace

Browse Source 
mutating webhook: audit log mutation existence and patch
This commit is contained in:
Kubernetes Prow Robot
2019-08-23 22:26:24 -07:00
committed by GitHub
parent 448049927b 98ad20cc8f
commit e2f57be0c0
20 changed files with 955 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
test/integration/master/BUILD
View File

@@ -27,6 +27,8 @@ go_test(
"//cmd/kube-apiserver/app/options:go_default_library",
"//cmd/kube-apiserver/app/testing:go_default_library",
"//pkg/master:go_default_library",
"//staging/src/k8s.io/api/admission/v1beta1:go_default_library",
"//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/networking/v1:go_default_library",
@@ -39,6 +41,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/audit/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library",

212
test/integration/master/audit_test.go
View File

@@ -20,23 +20,36 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"time"
"k8s.io/api/admission/v1beta1"
admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
"k8s.io/client-go/kubernetes"
clientset "k8s.io/client-go/kubernetes"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/kubernetes/test/utils"
"github.com/evanphx/json-patch"
jsonpatch "github.com/evanphx/json-patch"
)
const (
testWebhookConfigurationName = "auditmutation.integration.test"
testWebhookName = "auditmutation.integration.test"
)
var (
@@ -44,7 +57,7 @@ var (
apiVersion: {version}
kind: Policy
rules:
- level: RequestResponse
- level: {level}
resources:
- group: "" # core
resources: ["configmaps"]
@@ -163,20 +176,59 @@ rules:
// TestAudit ensures that both v1beta1 and v1 version audit api could work.
func TestAudit(t *testing.T) {
tcs := []struct {
auditLevel auditinternal.Level
enableMutatingWebhook bool
}{
{
auditLevel: auditinternal.LevelRequestResponse,
enableMutatingWebhook: false,
},
{
auditLevel: auditinternal.LevelMetadata,
enableMutatingWebhook: true,
},
{
auditLevel: auditinternal.LevelRequest,
enableMutatingWebhook: true,
},
{
auditLevel: auditinternal.LevelRequestResponse,
enableMutatingWebhook: true,
},
}
for version := range versions {
testAudit(t, version)
for _, tc := range tcs {
t.Run(fmt.Sprintf("%s.%s.%t", version, tc.auditLevel, tc.enableMutatingWebhook), func(t *testing.T) {
testAudit(t, version, tc.auditLevel, tc.enableMutatingWebhook)
})
}
}
}
func testAudit(t *testing.T, version string) {
func testAudit(t *testing.T, version string, level auditinternal.Level, enableMutatingWebhook bool) {
var url string
var err error
closeFunc := func() {}
if enableMutatingWebhook {
webhookMux := http.NewServeMux()
webhookMux.Handle("/mutation", utils.AdmissionWebhookHandler(t, admitFunc))
url, closeFunc, err = utils.NewAdmissionWebhookServer(webhookMux)
}
defer closeFunc()
if err != nil {
t.Fatalf("%v", err)
}
// prepare audit policy file
auditPolicy := []byte(strings.Replace(auditPolicyPattern, "{version}", version, 1))
auditPolicy := strings.Replace(auditPolicyPattern, "{version}", version, 1)
auditPolicy = strings.Replace(auditPolicy, "{level}", string(level), 1)
policyFile, err := ioutil.TempFile("", "audit-policy.yaml")
if err != nil {
t.Fatalf("Failed to create audit policy file: %v", err)
}
defer os.Remove(policyFile.Name())
if _, err := policyFile.Write(auditPolicy); err != nil {
if _, err := policyFile.Write([]byte(auditPolicy)); err != nil {
t.Fatalf("Failed to write audit policy file: %v", err)
}
if err := policyFile.Close(); err != nil {
@@ -205,21 +257,92 @@ func testAudit(t *testing.T, version string) {
t.Fatalf("Unexpected error: %v", err)
}
// perform configmap operations
configMapOperations(t, kubeclient)
if enableMutatingWebhook {
if err := createV1beta1MutationWebhook(kubeclient, url+"/mutation"); err != nil {
t.Fatal(err)
}
}
// check for corresponding audit logs
stream, err := os.Open(logFile.Name())
if err != nil {
t.Fatalf("Unexpected error: %v", err)
var lastMissingReport string
if err := wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
// perform configmap operations
configMapOperations(t, kubeclient)
// check for corresponding audit logs
stream, err := os.Open(logFile.Name())
if err != nil {
return false, fmt.Errorf("unexpected error: %v", err)
}
defer stream.Close()
missingReport, err := utils.CheckAuditLines(stream, getExpectedEvents(level, enableMutatingWebhook), versions[version])
if err != nil {
return false, fmt.Errorf("unexpected error: %v", err)
}
if len(missingReport.MissingEvents) > 0 {
lastMissingReport = missingReport.String()
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("failed to get expected events -- missingReport: %s, error: %v", lastMissingReport, err)
}
defer stream.Close()
missingReport, err := utils.CheckAuditLines(stream, expectedEvents, versions[version])
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
func getExpectedEvents(level auditinternal.Level, enableMutatingWebhook bool) []utils.AuditEvent {
if !enableMutatingWebhook {
return expectedEvents
}
if len(missingReport.MissingEvents) > 0 {
t.Errorf(missingReport.String())
var webhookMutationAnnotations, webhookPatchAnnotations map[string]string
var requestObject, responseObject bool
if level.GreaterOrEqual(auditinternal.LevelMetadata) {
// expect mutation existence annotation
webhookMutationAnnotations = map[string]string{}
webhookMutationAnnotations[mutating.MutationAuditAnnotationPrefix+"round_0_index_0"] = fmt.Sprintf(`{"configuration":"%s","webhook":"%s","mutated":%t}`, testWebhookConfigurationName, testWebhookName, true)
}
if level.GreaterOrEqual(auditinternal.LevelRequest) {
// expect actual patch annotation
webhookPatchAnnotations = map[string]string{}
webhookPatchAnnotations[mutating.PatchAuditAnnotationPrefix+"round_0_index_0"] = strings.Replace(fmt.Sprintf(`{"configuration": "%s", "webhook": "%s", "patch": %s, "patchType": "JSONPatch"}`, testWebhookConfigurationName, testWebhookName, `[{"op":"add","path":"/data","value":{"test":"dummy"}}]`), " ", "", -1)
// expect request object in audit log
requestObject = true
}
if level.GreaterOrEqual(auditinternal.LevelRequestResponse) {
// expect response obect in audit log
responseObject = true
}
return []utils.AuditEvent{
{
// expect CREATE audit event with webhook in effect
Level: level,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", namespace),
Verb: "create",
Code: 201,
User: auditTestUser,
Resource: "configmaps",
Namespace: namespace,
AuthorizeDecision: "allow",
RequestObject: requestObject,
ResponseObject: responseObject,
AdmissionWebhookMutationAnnotations: webhookMutationAnnotations,
AdmissionWebhookPatchAnnotations: webhookPatchAnnotations,
}, {
// expect UPDATE audit event with webhook in effect
Level: level,
Stage: auditinternal.StageResponseComplete,
RequestURI: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/audit-configmap", namespace),
Verb: "update",
Code: 200,
User: auditTestUser,
Resource: "configmaps",
Namespace: namespace,
AuthorizeDecision: "allow",
RequestObject: requestObject,
ResponseObject: responseObject,
AdmissionWebhookMutationAnnotations: webhookMutationAnnotations,
AdmissionWebhookPatchAnnotations: webhookPatchAnnotations,
},
}
}
@@ -269,3 +392,56 @@ func expectNoError(t *testing.T, err error, msg string) {
t.Fatalf("%s: %v", msg, err)
}
}
func admitFunc(review *v1beta1.AdmissionReview) error {
gvk := schema.GroupVersionKind{Group: "admission.k8s.io", Version: "v1beta1", Kind: "AdmissionReview"}
if review.GetObjectKind().GroupVersionKind() != gvk {
return fmt.Errorf("invalid admission review kind: %#v", review.GetObjectKind().GroupVersionKind())
}
if len(review.Request.Object.Raw) > 0 {
u := &unstructured.Unstructured{Object: map[string]interface{}{}}
if err := json.Unmarshal(review.Request.Object.Raw, u); err != nil {
return fmt.Errorf("failed to deserialize object: %s with error: %v", string(review.Request.Object.Raw), err)
}
review.Request.Object.Object = u
}
if len(review.Request.OldObject.Raw) > 0 {
u := &unstructured.Unstructured{Object: map[string]interface{}{}}
if err := json.Unmarshal(review.Request.OldObject.Raw, u); err != nil {
return fmt.Errorf("failed to deserialize object: %s with error: %v", string(review.Request.OldObject.Raw), err)
}
review.Request.OldObject.Object = u
}
review.Response = &v1beta1.AdmissionResponse{
Allowed: true,
UID: review.Request.UID,
Result: &metav1.Status{Message: "admitted"},
}
review.Response.Patch = []byte(`[{"op":"add","path":"/data","value":{"test":"dummy"}}]`)
jsonPatch := v1beta1.PatchTypeJSONPatch
review.Response.PatchType = &jsonPatch
return nil
}
func createV1beta1MutationWebhook(client clientset.Interface, endpoint string) error {
fail := admissionv1beta1.Fail
// Attaching Mutation webhook to API server
_, err := client.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Create(&admissionv1beta1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: testWebhookConfigurationName},
Webhooks: []admissionv1beta1.MutatingWebhook{{
Name: testWebhookName,
ClientConfig: admissionv1beta1.WebhookClientConfig{
URL: &endpoint,
CABundle: utils.LocalhostCert,
},
Rules: []admissionv1beta1.RuleWithOperations{{
Operations: []admissionv1beta1.OperationType{admissionv1beta1.Create, admissionv1beta1.Update},
Rule: admissionv1beta1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}},
}},
FailurePolicy: &fail,
AdmissionReviewVersions: []string{"v1beta1"},
}},
})
return err
}