diff --git a/pkg/registry/flowcontrol/rest/BUILD b/pkg/registry/flowcontrol/rest/BUILD index ce2780d854a..f1fa23bec26 100644 --- a/pkg/registry/flowcontrol/rest/BUILD +++ b/pkg/registry/flowcontrol/rest/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -11,10 +11,16 @@ go_library( "//pkg/registry/flowcontrol/flowschema/storage:go_default_library", "//pkg/registry/flowcontrol/prioritylevelconfiguration/storage:go_default_library", "//staging/src/k8s.io/api/flowcontrol/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) @@ -31,3 +37,15 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_default_test", + srcs = ["storage_flowcontrol_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/api/flowcontrol/v1alpha1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], +) diff --git a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go index 1565ef8b132..15fd7758c3a 100644 --- a/pkg/registry/flowcontrol/rest/storage_flowcontrol.go +++ b/pkg/registry/flowcontrol/rest/storage_flowcontrol.go @@ -17,20 +17,34 @@ limitations under the License. package rest import ( + "fmt" + "time" + flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + flowcontrolbootstrap "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" + flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/flowcontrol" flowschemastore "k8s.io/kubernetes/pkg/registry/flowcontrol/flowschema/storage" prioritylevelconfigurationstore "k8s.io/kubernetes/pkg/registry/flowcontrol/prioritylevelconfiguration/storage" ) -// RESTStorageProvider implements +var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{} + +// RESTStorageProvider is a provider of REST storage type RESTStorageProvider struct{} +// PostStartHookName is the name of the post-start-hook provided by flow-control storage +const PostStartHookName = "apiserver/bootstrap-system-flowcontrol-configuration" + // NewRESTStorage creates a new rest storage for flow-control api models. func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(flowcontrol.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) @@ -71,3 +85,129 @@ func (p RESTStorageProvider) v1alpha1Storage(apiResourceConfigSource serverstora func (p RESTStorageProvider) GroupName() string { return flowcontrol.GroupName } + +func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) { + return PostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error { + flowcontrolClientSet := flowcontrolclient.NewForConfigOrDie(hookContext.LoopbackClientConfig) + go func() { + const retryCreatingSuggestedSettingsInterval = time.Second + _ = wait.PollImmediateUntil( + retryCreatingSuggestedSettingsInterval, + func() (bool, error) { + shouldEnsureSuggested, err := shouldEnsureAllPredefined(flowcontrolClientSet) + if err != nil { + klog.Errorf("failed getting exempt flow-schema, will retry later: %v", err) + return false, nil + } + if shouldEnsureSuggested { + err := ensure( + flowcontrolClientSet, + flowcontrolbootstrap.SuggestedFlowSchemas, + flowcontrolbootstrap.SuggestedPriorityLevelConfigurations) + if err != nil { + klog.Errorf("failed ensuring suggested settings, will retry later: %v", err) + return false, nil + } + } + return true, nil + }, + hookContext.StopCh) + const retryCreatingMandatorySettingsInterval = time.Minute + _ = wait.PollImmediateUntil( + retryCreatingMandatorySettingsInterval, + func() (bool, error) { + if err := upgrade( + flowcontrolClientSet, + flowcontrolbootstrap.MandatoryFlowSchemas, + // Note: the "exempt" priority-level is supposed tobe the last item in the pre-defined + // list, so that a crash in the midst of the first kube-apiserver startup does not prevent + // the full initial set of objects from being created. + flowcontrolbootstrap.MandatoryPriorityLevelConfigurations, + ); err != nil { + klog.Errorf("failed creating default flowcontrol settings: %v", err) + return false, nil + } + return false, nil // always retry + }, + hookContext.StopCh) + }() + return nil + }, nil + +} + +// Returns false if there's a "exempt" priority-level existing in the cluster, otherwise returns a true +// if the "exempt" priority-level is not found. +func shouldEnsureAllPredefined(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alpha1Interface) (bool, error) { + if _, err := flowcontrolClientSet.PriorityLevelConfigurations().Get(flowcontrol.PriorityLevelConfigurationNameExempt, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil +} + +func ensure(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alpha1Interface, flowSchemas []*flowcontrolv1alpha1.FlowSchema, priorityLevels []*flowcontrolv1alpha1.PriorityLevelConfiguration) error { + for _, flowSchema := range flowSchemas { + _, err := flowcontrolClientSet.FlowSchemas().Create(flowSchema) + if apierrors.IsAlreadyExists(err) { + klog.V(3).Infof("system preset FlowSchema %s already exists, skipping creating", flowSchema.Name) + continue + } + if err != nil { + return fmt.Errorf("cannot create FlowSchema %s due to %v", flowSchema.Name, err) + } + klog.V(3).Infof("created system preset FlowSchema %s", flowSchema.Name) + } + for _, priorityLevelConfiguration := range priorityLevels { + _, err := flowcontrolClientSet.PriorityLevelConfigurations().Create(priorityLevelConfiguration) + if apierrors.IsAlreadyExists(err) { + klog.V(3).Infof("system preset PriorityLevelConfiguration %s already exists, skipping creating", priorityLevelConfiguration.Name) + continue + } + if err != nil { + return fmt.Errorf("cannot create PriorityLevelConfiguration %s due to %v", priorityLevelConfiguration.Name, err) + } + klog.V(3).Infof("created system preset PriorityLevelConfiguration %s", priorityLevelConfiguration.Name) + } + return nil +} + +func upgrade(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alpha1Interface, flowSchemas []*flowcontrolv1alpha1.FlowSchema, priorityLevels []*flowcontrolv1alpha1.PriorityLevelConfiguration) error { + for _, flowSchema := range flowSchemas { + _, err := flowcontrolClientSet.FlowSchemas().Get(flowSchema.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed getting FlowSchema %s due to %v, will retry later", flowSchema.Name, err) + } + // TODO(yue9944882): extract existing version from label and compare + // TODO(yue9944882): create w/ version string attached + _, err = flowcontrolClientSet.FlowSchemas().Create(flowSchema) + if apierrors.IsAlreadyExists(err) { + klog.V(3).Infof("system preset FlowSchema %s already exists, skipping creating", flowSchema.Name) + continue + } + if err != nil { + return fmt.Errorf("cannot create FlowSchema %s due to %v", flowSchema.Name, err) + } + klog.V(3).Infof("created system preset FlowSchema %s", flowSchema.Name) + } + for _, priorityLevelConfiguration := range priorityLevels { + _, err := flowcontrolClientSet.FlowSchemas().Get(priorityLevelConfiguration.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed getting PriorityLevelConfiguration %s due to %v, will retry later", priorityLevelConfiguration.Name, err) + } + // TODO(yue9944882): extract existing version from label and compare + // TODO(yue9944882): create w/ version string attached + _, err = flowcontrolClientSet.PriorityLevelConfigurations().Create(priorityLevelConfiguration) + if apierrors.IsAlreadyExists(err) { + klog.V(3).Infof("system preset PriorityLevelConfiguration %s already exists, skipping creating", priorityLevelConfiguration.Name) + continue + } + if err != nil { + return fmt.Errorf("cannot create PriorityLevelConfiguration %s due to %v", priorityLevelConfiguration.Name, err) + } + klog.V(3).Infof("created system preset PriorityLevelConfiguration %s", priorityLevelConfiguration.Name) + } + return nil +} diff --git a/pkg/registry/flowcontrol/rest/storage_flowcontrol_test.go b/pkg/registry/flowcontrol/rest/storage_flowcontrol_test.go new file mode 100644 index 00000000000..015dc1a9041 --- /dev/null +++ b/pkg/registry/flowcontrol/rest/storage_flowcontrol_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + flowcontrol "k8s.io/api/flowcontrol/v1alpha1" + "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" + "k8s.io/client-go/kubernetes/fake" +) + +func TestShouldEnsurePredefinedSettings(t *testing.T) { + testCases := []struct { + name string + existingPriorityLevel *flowcontrol.PriorityLevelConfiguration + expected bool + }{ + { + name: "should ensure if exempt priority-level is absent", + existingPriorityLevel: nil, + expected: true, + }, + { + name: "should not ensure if exempt priority-level is present", + existingPriorityLevel: bootstrap.MandatoryPriorityLevelConfigurationExempt, + expected: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + c := fake.NewSimpleClientset() + if testCase.existingPriorityLevel != nil { + c.FlowcontrolV1alpha1().PriorityLevelConfigurations().Create(testCase.existingPriorityLevel) + } + should, err := shouldEnsureAllPredefined(c.FlowcontrolV1alpha1()) + assert.NoError(t, err) + assert.Equal(t, testCase.expected, should) + }) + } +} diff --git a/staging/src/k8s.io/apiserver/BUILD b/staging/src/k8s.io/apiserver/BUILD index 72460f0ed2a..894b53fa584 100644 --- a/staging/src/k8s.io/apiserver/BUILD +++ b/staging/src/k8s.io/apiserver/BUILD @@ -15,6 +15,7 @@ filegroup( "//staging/src/k8s.io/apiserver/pkg/apis/config:all-srcs", "//staging/src/k8s.io/apiserver/pkg/apis/example:all-srcs", "//staging/src/k8s.io/apiserver/pkg/apis/example2:all-srcs", + "//staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap:all-srcs", "//staging/src/k8s.io/apiserver/pkg/audit:all-srcs", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:all-srcs", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:all-srcs", diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/BUILD new file mode 100644 index 00000000000..4c092ec41c4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["default.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap", + importpath = "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/coordination/v1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/flowcontrol/v1alpha1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go new file mode 100644 index 00000000000..a3be49dd5e5 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go @@ -0,0 +1,435 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + flowcontrol "k8s.io/api/flowcontrol/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/apiserver/pkg/authentication/user" +) + +// The objects that define an apiserver's initial behavior +var ( + MandatoryPriorityLevelConfigurations = []*flowcontrol.PriorityLevelConfiguration{ + MandatoryPriorityLevelConfigurationExempt, + MandatoryPriorityLevelConfigurationCatchAll, + } + MandatoryFlowSchemas = []*flowcontrol.FlowSchema{ + MandatoryFlowSchemaExempt, + MandatoryFlowSchemaCatchAll, + } +) + +// The objects that define an apiserver's initial behavior +var ( + SuggestedPriorityLevelConfigurations = []*flowcontrol.PriorityLevelConfiguration{ + // "system" priority-level is for the system components that affects self-maintenance of the + // cluster and the availability of those running pods in the cluster, including kubelet and + // kube-proxy. + SuggestedPriorityLevelConfigurationSystem, + // "leader-election" is dedicated for controllers' leader-election, which majorly affects the + // availability of any controller runs in the cluster. + SuggestedPriorityLevelConfigurationLeaderElection, + // "workload-high" is used by those workloads with higher priority but their failure won't directly + // impact the existing running pods in the cluster, which includes kube-scheduler, and those well-known + // built-in workloads such as "deployments", "replicasets" and other low-level custom workload which + // is important for the cluster. + SuggestedPriorityLevelConfigurationWorkloadHigh, + // "workload-low" is used by those workloads with lower priority which availability only has a + // minor impact on the cluster. + SuggestedPriorityLevelConfigurationWorkloadLow, + } + SuggestedFlowSchemas = []*flowcontrol.FlowSchema{ + SuggestedFlowSchemaSystemNodes, // references "system" priority-level + SuggestedFlowSchemaSystemLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaWorkloadLeaderElection, // references "leader-election" priority-level + SuggestedFlowSchemaKubeControllerManager, // references "workload-high" priority-level + SuggestedFlowSchemaKubeScheduler, // references "workload-high" priority-level + SuggestedFlowSchemaKubeSystemServiceAccounts, // references "workload-high" priority-level + } +) + +// Mandatory PriorityLevelConfiguration objects +var ( + MandatoryPriorityLevelConfigurationExempt = newPriorityLevelConfiguration( + flowcontrol.PriorityLevelConfigurationNameExempt, + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementExempt, + }, + ) + MandatoryPriorityLevelConfigurationCatchAll = newPriorityLevelConfiguration( + "catch-all", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + AssuredConcurrencyShares: 100, + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 100, + }, + }, + }, + }) +) + +// Mandatory FlowSchema objects +var ( + // exempt priority-level + MandatoryFlowSchemaExempt = newFlowSchema( + "exempt", + flowcontrol.PriorityLevelConfigurationNameExempt, + 0, // matchingPrecedence + "", // distinguisherMethodType + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.SystemPrivilegedGroup), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true, + ), + }, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}, + ), + }, + }, + ) + // catch-all priority-level + MandatoryFlowSchemaCatchAll = newFlowSchema( + "catch-all", + "catch-all", + 10000, // matchingPrecedence + flowcontrol.FlowDistinguisherMethodByUserType, // distinguisherMethodType + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.AllUnauthenticated, user.AllAuthenticated), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true, + ), + }, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}, + ), + }, + }, + ) +) + +// Suggested PriorityLevelConfiguration objects +var ( + // system priority-level + SuggestedPriorityLevelConfigurationSystem = newPriorityLevelConfiguration( + "system", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + AssuredConcurrencyShares: 30, + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 64, + HandSize: 6, + QueueLengthLimit: 1000, + }, + }, + }, + }) + // leader-election priority-level + SuggestedPriorityLevelConfigurationLeaderElection = newPriorityLevelConfiguration( + "leader-election", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + AssuredConcurrencyShares: 10, + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 16, + HandSize: 4, + QueueLengthLimit: 100, + }, + }, + }, + }) + // workload-high priority-level + SuggestedPriorityLevelConfigurationWorkloadHigh = newPriorityLevelConfiguration( + "workload-high", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + AssuredConcurrencyShares: 40, + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 100, + }, + }, + }, + }) + // workload-low priority-level + SuggestedPriorityLevelConfigurationWorkloadLow = newPriorityLevelConfiguration( + "workload-low", + flowcontrol.PriorityLevelConfigurationSpec{ + Type: flowcontrol.PriorityLevelEnablementLimited, + Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ + AssuredConcurrencyShares: 20, + LimitResponse: flowcontrol.LimitResponse{ + Type: flowcontrol.LimitResponseTypeQueue, + Queuing: &flowcontrol.QueuingConfiguration{ + Queues: 128, + HandSize: 6, + QueueLengthLimit: 100, + }, + }, + }, + }) +) + +// Suggested FlowSchema objects +var ( + SuggestedFlowSchemaSystemNodes = newFlowSchema( + "system-nodes", "system", 1500, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(user.NodesGroup), // the nodes group + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaSystemLeaderElection = newFlowSchema( + "system-leader-election", "leader-election", 2500, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: append( + users(user.KubeControllerManager, user.KubeScheduler), + kubeSystemServiceAccount(flowcontrol.NameAll)...), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{"get", "create", "update"}, + []string{corev1.GroupName}, + []string{"endpoints", "configmaps"}, + []string{"kube-system"}, + false), + resourceRule( + []string{"get", "create", "update"}, + []string{coordinationv1.GroupName}, + []string{"leases"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + SuggestedFlowSchemaWorkloadLeaderElection = newFlowSchema( + "workload-leader-election", "leader-election", 2500, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: kubeSystemServiceAccount(flowcontrol.NameAll), + ResourceRules: []flowcontrol.ResourcePolicyRule{ + resourceRule( + []string{flowcontrol.VerbAll}, + []string{corev1.GroupName}, + []string{"endpoints", "configmaps"}, + []string{flowcontrol.NamespaceEvery}, + false), + resourceRule( + []string{flowcontrol.VerbAll}, + []string{coordinationv1.GroupName}, + []string{"leases"}, + []string{flowcontrol.NamespaceEvery}, + false), + }, + }, + ) + SuggestedFlowSchemaKubeControllerManager = newFlowSchema( + "kube-controller-manager", "workload-high", 3500, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: users(user.KubeControllerManager), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaKubeScheduler = newFlowSchema( + "kube-scheduler", "workload-high", 3500, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: users(user.KubeScheduler), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaKubeSystemServiceAccounts = newFlowSchema( + "kube-system-service-accounts", "workload-high", 3500, + flowcontrol.FlowDistinguisherMethodByNamespaceType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: kubeSystemServiceAccount(flowcontrol.NameAll), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) + SuggestedFlowSchemaServiceAccounts = newFlowSchema( + "service-accounts", "workload-low", 7500, + flowcontrol.FlowDistinguisherMethodByUserType, + flowcontrol.PolicyRulesWithSubjects{ + Subjects: groups(serviceaccount.AllServiceAccountsGroup), + ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.APIGroupAll}, + []string{flowcontrol.ResourceAll}, + []string{flowcontrol.NamespaceEvery}, + true)}, + NonResourceRules: []flowcontrol.NonResourcePolicyRule{ + nonResourceRule( + []string{flowcontrol.VerbAll}, + []string{flowcontrol.NonResourceAll}), + }, + }, + ) +) + +func newPriorityLevelConfiguration(name string, spec flowcontrol.PriorityLevelConfigurationSpec) *flowcontrol.PriorityLevelConfiguration { + return &flowcontrol.PriorityLevelConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: spec} +} + +func newFlowSchema(name, plName string, matchingPrecedence int32, dmType flowcontrol.FlowDistinguisherMethodType, rules ...flowcontrol.PolicyRulesWithSubjects) *flowcontrol.FlowSchema { + var dm *flowcontrol.FlowDistinguisherMethod + if dmType != "" { + dm = &flowcontrol.FlowDistinguisherMethod{Type: dmType} + } + return &flowcontrol.FlowSchema{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: flowcontrol.FlowSchemaSpec{ + PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{ + Name: plName, + }, + MatchingPrecedence: matchingPrecedence, + DistinguisherMethod: dm, + Rules: rules}, + } + +} + +func groups(names ...string) []flowcontrol.Subject { + ans := make([]flowcontrol.Subject, len(names)) + for idx, name := range names { + ans[idx] = flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindGroup, + Group: &flowcontrol.GroupSubject{ + Name: name, + }, + } + } + return ans +} + +func users(names ...string) []flowcontrol.Subject { + ans := make([]flowcontrol.Subject, len(names)) + for idx, name := range names { + ans[idx] = flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindUser, + User: &flowcontrol.UserSubject{ + Name: name, + }, + } + } + return ans +} + +func kubeSystemServiceAccount(names ...string) []flowcontrol.Subject { + subjects := []flowcontrol.Subject{} + for _, name := range names { + subjects = append(subjects, flowcontrol.Subject{ + Kind: flowcontrol.SubjectKindServiceAccount, + ServiceAccount: &flowcontrol.ServiceAccountSubject{ + Name: name, + Namespace: metav1.NamespaceSystem, + }, + }) + } + return subjects +} + +func resourceRule(verbs []string, groups []string, resources []string, namespaces []string, clusterScoped bool) flowcontrol.ResourcePolicyRule { + return flowcontrol.ResourcePolicyRule{ + Verbs: verbs, + APIGroups: groups, + Resources: resources, + Namespaces: namespaces, + ClusterScope: clusterScoped, + } +} + +func nonResourceRule(verbs []string, nonResourceURLs []string) flowcontrol.NonResourcePolicyRule { + return flowcontrol.NonResourcePolicyRule{Verbs: verbs, NonResourceURLs: nonResourceURLs} +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 9ca0f7a2bfc..212ce93807c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1286,6 +1286,7 @@ k8s.io/apiserver/pkg/apis/config/v1 k8s.io/apiserver/pkg/apis/config/validation k8s.io/apiserver/pkg/apis/example k8s.io/apiserver/pkg/apis/example/v1 +k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap k8s.io/apiserver/pkg/audit k8s.io/apiserver/pkg/audit/event k8s.io/apiserver/pkg/audit/policy