github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-12-05 07:26:56 +00:00
Code Issues Projects Releases Wiki Activity

Implement authz config file reloading

Browse Source
This commit is contained in:
Jordan Liggitt
2023-11-08 08:49:58 -06:00
parent 3a98e60a71
commit 5dc92ada06
10 changed files with 744 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
test/integration/auth/authz_config_test.go
View File

@@ -25,6 +25,8 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
@@ -32,7 +34,9 @@ import (
authorizationv1 "k8s.io/api/authorization/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/features"
authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@@ -113,6 +117,8 @@ authorizers:
}
func TestMultiWebhookAuthzConfig(t *testing.T) {
authzmetrics.ResetMetricsForTest()
defer authzmetrics.ResetMetricsForTest()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
dir := t.TempDir()
@@ -235,14 +241,36 @@ users:
t.Fatal(err)
}
// returns an allow response when called
serverAllowReloadedCalled := atomic.Int32{}
serverAllowReloaded := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
serverAllowReloadedCalled.Add(1)
sar := &authorizationv1.SubjectAccessReview{}
if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
t.Error(err)
}
t.Log("serverAllowReloaded", sar)
sar.Status.Allowed = true
sar.Status.Reason = "allowed2 by webhook"
if err := json.NewEncoder(w).Encode(sar); err != nil {
t.Error(err)
}
}))
defer serverAllowReloaded.Close()
serverAllowReloadedKubeconfigName := filepath.Join(dir, "serverAllowReloaded.yaml")
if err := os.WriteFile(serverAllowReloadedKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverAllowReloaded.URL)), os.FileMode(0644)); err != nil {
t.Fatal(err)
}
resetCounts := func() {
serverErrorCalled.Store(0)
serverTimeoutCalled.Store(0)
serverDenyCalled.Store(0)
serverNoOpinionCalled.Store(0)
serverAllowCalled.Store(0)
serverAllowReloadedCalled.Store(0)
}
assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount int32) {
assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount int32) {
t.Helper()
if e, a := errorCount, serverErrorCalled.Load(); e != a {
t.Errorf("expected fail webhook calls: %d, got %d", e, a)
@@ -259,6 +287,9 @@ users:
if e, a := allowCount, serverAllowCalled.Load(); e != a {
t.Errorf("expected allow webhook calls: %d, got %d", e, a)
}
if e, a := allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
t.Errorf("expected allowReloaded webhook calls: %d, got %d", e, a)
}
resetCounts()
}
@@ -274,6 +305,8 @@ authorizers:
failurePolicy: Deny
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverErrorKubeconfigName+`
@@ -289,6 +322,8 @@ authorizers:
failurePolicy: Deny
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverTimeoutKubeconfigName+`
@@ -304,6 +339,8 @@ authorizers:
failurePolicy: NoOpinion
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverDenyKubeconfigName+`
@@ -317,6 +354,8 @@ authorizers:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverNoOpinionKubeconfigName+`
@@ -327,6 +366,8 @@ authorizers:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverAllowKubeconfigName+`
@@ -362,7 +403,7 @@ authorizers:
t.Fatal("expected denied, got allowed")
} else {
t.Log(result.Status.Reason)
assertCounts(1, 0, 0, 0, 0)
assertCounts(1, 0, 0, 0, 0, 0)
}
// timeout webhook short circuits
@@ -383,7 +424,7 @@ authorizers:
t.Fatal("expected denied, got allowed")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 1, 0, 0, 0)
assertCounts(0, 1, 0, 0, 0, 0)
}
// deny webhook short circuits
@@ -404,7 +445,7 @@ authorizers:
t.Fatal("expected denied, got allowed")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 0, 1, 0, 0)
assertCounts(0, 0, 1, 0, 0, 0)
}
// no-opinion webhook passes through, allow webhook allows
@@ -425,6 +466,231 @@ authorizers:
t.Fatal("expected allowed, got denied")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 0, 0, 1, 1)
assertCounts(0, 0, 0, 1, 1, 0)
}
// check last loaded success/failure metric timestamps, ensure success is present, failure is not
initialReloadSuccess, initialReloadFailure, err := getReloadTimes(t, adminClient)
if err != nil {
t.Fatal(err)
}
if initialReloadSuccess == nil {
t.Fatal("expected success timestamp, got none")
}
if initialReloadFailure != nil {
t.Fatal("expected no failure timestamp, got one")
}
// write bogus file
if err := os.WriteFile(configFileName, []byte(`apiVersion: apiserver.config.k8s.io`), os.FileMode(0644)); err != nil {
t.Fatal(err)
}
// wait for failure timestamp > success timestamp
var reload1Success, reload1Failure *time.Time
err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
reload1Success, reload1Failure, err = getReloadTimes(t, adminClient)
if err != nil {
t.Fatal(err)
}
if reload1Success == nil {
t.Fatal("expected success timestamp, got none")
}
if !reload1Success.Equal(*initialReloadSuccess) {
t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialReloadSuccess.String(), reload1Success.String())
}
if reload1Failure == nil {
t.Log("expected failure timestamp, got nil, retrying")
return false, nil
}
if !reload1Failure.After(*reload1Success) {
t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Failure.String(), reload1Success.String())
}
return true, nil
})
if err != nil {
t.Fatal(err)
}
// ensure authz still works
t.Log("checking allow")
if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
User: "alice",
ResourceAttributes: &authorizationv1.ResourceAttributes{
Verb: "list",
Group: "",
Version: "v1",
Resource: "configmaps",
Namespace: "allow",
Name: "",
},
}}, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
} else if !result.Status.Allowed {
t.Fatal("expected allowed, got denied")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 0, 0, 1, 1, 0)
}
// write good config with different webhook
if err := os.WriteFile(configFileName, []byte(`
apiVersion: apiserver.config.k8s.io/v1alpha1
kind: AuthorizationConfiguration
authorizers:
- type: Webhook
name: allowreloaded.example.com
webhook:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: `+serverAllowReloadedKubeconfigName+`
`), os.FileMode(0644)); err != nil {
t.Fatal(err)
}
// wait for success timestamp > reload1Failure timestamp
var reload2Success, reload2Failure *time.Time
err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
reload2Success, reload2Failure, err = getReloadTimes(t, adminClient)
if err != nil {
t.Fatal(err)
}
if reload2Failure == nil {
t.Log("expected failure timestamp, got nil, retrying")
return false, nil
}
if !reload2Failure.Equal(*reload1Failure) {
t.Fatalf("failure timestamp changed from reload1Failure %s to %s unexpectedly", reload1Failure.String(), reload2Failure.String())
}
if reload2Success == nil {
t.Fatal("expected success timestamp, got none")
}
if reload2Success.Equal(*initialReloadSuccess) {
t.Log("success timestamp hasn't updated from initial success, retrying")
return false, nil
}
if !reload2Success.After(*reload2Failure) {
t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Success.String(), reload2Failure.String())
}
return true, nil
})
if err != nil {
t.Fatal(err)
}
// ensure authz still works, new webhook is called
t.Log("checking allow")
if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
User: "alice",
ResourceAttributes: &authorizationv1.ResourceAttributes{
Verb: "list",
Group: "",
Version: "v1",
Resource: "configmaps",
Namespace: "allow",
Name: "",
},
}}, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
} else if !result.Status.Allowed {
t.Fatal("expected allowed, got denied")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 0, 0, 0, 0, 1)
}
// delete file (do this test last because it makes file watch fall back to one minute poll interval)
if err := os.Remove(configFileName); err != nil {
t.Fatal(err)
}
// wait for failure timestamp > success timestamp
var reload3Success, reload3Failure *time.Time
err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
reload3Success, reload3Failure, err = getReloadTimes(t, adminClient)
if err != nil {
t.Fatal(err)
}
if reload3Success == nil {
t.Fatal("expected success timestamp, got none")
}
if !reload3Success.Equal(*reload2Success) {
t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Success.String(), reload3Success.String())
}
if reload3Failure == nil {
t.Log("expected failure timestamp, got nil, retrying")
return false, nil
}
if reload3Failure.Equal(*reload2Failure) {
t.Log("failure timestamp hasn't updated, retrying")
return false, nil
}
if !reload3Failure.After(*reload3Success) {
t.Fatalf("expected failure timestamp to be more recent than success, got %s <= %s", reload3Failure.String(), reload3Success.String())
}
return true, nil
})
if err != nil {
t.Fatal(err)
}
// ensure authz still works, new webhook is called
t.Log("checking allow")
if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
User: "alice",
ResourceAttributes: &authorizationv1.ResourceAttributes{
Verb: "list",
Group: "",
Version: "v1",
Resource: "configmaps",
Namespace: "allow",
Name: "",
},
}}, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
} else if !result.Status.Allowed {
t.Fatal("expected allowed, got denied")
} else {
t.Log(result.Status.Reason)
assertCounts(0, 0, 0, 0, 0, 1)
}
}
func getReloadTimes(t *testing.T, client *clientset.Clientset) (*time.Time, *time.Time, error) {
data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09
if err != nil {
return nil, nil, err
}
var success, failure *time.Time
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") {
t.Log(line)
values := strings.Split(line, " ")
value, err := strconv.ParseFloat(values[len(values)-1], 64)
if err != nil {
return nil, nil, err
}
seconds := int64(value)
nanoseconds := int64((value - float64(seconds)) * 1000000000)
tm := time.Unix(seconds, nanoseconds)
if strings.Contains(line, `"success"`) {
success = &tm
t.Log("success", success.String())
}
if strings.Contains(line, `"failure"`) {
failure = &tm
t.Log("failure", failure.String())
}
}
}
return success, failure, nil
}