diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go
index a3a7b03442d..5a1c3ec9734 100644
--- a/pkg/controlplane/apiserver/options/options.go
+++ b/pkg/controlplane/apiserver/options/options.go
@@ -222,6 +222,9 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple
 		klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost)
 	}
 
+	// put authorization options in final state
+	completed.Authorization.Complete()
+	// adjust authentication for completed authorization
 	completed.Authentication.ApplyAuthorization(completed.Authorization)
 
 	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
diff --git a/pkg/kubeapiserver/options/authorization.go b/pkg/kubeapiserver/options/authorization.go
index 4e9e24fb729..02d8675e549 100644
--- a/pkg/kubeapiserver/options/authorization.go
+++ b/pkg/kubeapiserver/options/authorization.go
@@ -80,7 +80,7 @@ type BuiltInAuthorizationOptions struct {
 // NewBuiltInAuthorizationOptions create a BuiltInAuthorizationOptions with default value
 func NewBuiltInAuthorizationOptions() *BuiltInAuthorizationOptions {
 	return &BuiltInAuthorizationOptions{
-		Modes:                       []string{authzmodes.ModeAlwaysAllow},
+		Modes:                       []string{},
 		WebhookVersion:              "v1beta1",
 		WebhookCacheAuthorizedTTL:   5 * time.Minute,
 		WebhookCacheUnauthorizedTTL: 30 * time.Second,
@@ -88,6 +88,14 @@ func NewBuiltInAuthorizationOptions() *BuiltInAuthorizationOptions {
 	}
 }
 
+// Complete modifies authorization options
+func (o *BuiltInAuthorizationOptions) Complete() []error {
+	if len(o.AuthorizationConfigurationFile) == 0 && len(o.Modes) == 0 {
+		o.Modes = []string{authzmodes.ModeAlwaysAllow}
+	}
+	return nil
+}
+
 // Validate checks invalid config combination
 func (o *BuiltInAuthorizationOptions) Validate() []error {
 	if o == nil {
@@ -185,7 +193,7 @@ func (o *BuiltInAuthorizationOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 
 	fs.StringSliceVar(&o.Modes, authorizationModeFlag, o.Modes, ""+
-		"Ordered list of plug-ins to do authorization on secure port. Comma-delimited list of: "+
+		"Ordered list of plug-ins to do authorization on secure port. Defaults to AlwaysAllow if --authorization-config is not used. Comma-delimited list of: "+
 		strings.Join(authzmodes.AuthorizationModeChoices, ",")+".")
 
 	fs.StringVar(&o.PolicyFile, authorizationPolicyFileFlag, o.PolicyFile, ""+