github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 23:15:14 +00:00
Code Issues Projects Releases Wiki Activity

kubeadm: Enable the Node Authorizer in v1.7 and fix some small bugs related to the enablement

Browse Source
This commit is contained in:
Lucas Käldström 2017-06-07 21:07:49 +03:00
parent cc568f6433
commit b7700ef4dd
No known key found for this signature in database
GPG Key ID: 3FA3783D77751514
12 changed files with 176 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/kubeadm/app/cmd/BUILD
View File

@ -41,6 +41,7 @@ go_library(
"//cmd/kubeadm/app/util/token:go_default_library",
"//pkg/api:go_default_library",
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/printers:go_default_library",
"//pkg/util/i18n:go_default_library",
@ -54,6 +55,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
@ -65,6 +67,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
"defaults_test.go",
"reset_test.go",
"token_test.go",
],
@ -73,6 +76,7 @@ go_test(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//pkg/util/version:go_default_library",
],
)

18
cmd/kubeadm/app/cmd/defaults.go
View File

@ -21,10 +21,12 @@ import (
"net"
netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/util/version"
)
@ -54,8 +56,12 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
}
// Defaulting is made here because it's dependent on the version currently, which is determined above
// TODO(luxas): Cleanup this once we have dropped v1.6 support and move this code into the API group defaulting
cfg.AuthorizationModes = defaultAuthorizationModes(cfg.AuthorizationModes, k8sVersion)
fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes)
fmt.Printf("[init] Using Authorization modes: %v\n", cfg.AuthorizationModes)
// Warn about the limitations with the current cloudprovider solution.
if cfg.CloudProvider != "" {
@ -73,3 +79,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
return nil
}
func defaultAuthorizationModes(authzModes []string, k8sVersion *version.Version) []string {
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
strset := sets.NewString(authzModes...)
if !strset.Has(authzmodes.ModeNode) {
return append([]string{authzmodes.ModeNode}, authzModes...)
}
}
return authzModes
}

54
cmd/kubeadm/app/cmd/defaults_test.go Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"strings"
"testing"
"k8s.io/kubernetes/pkg/util/version"
)
func TestDefaultAuthorizationModes(t *testing.T) {
var tests = []struct {
authzModes []string
version string
expected []string
}{
{[]string{"RBAC"}, "v1.6.0", []string{"RBAC"}},
{[]string{"RBAC", "ABAC"}, "v1.6.4", []string{"RBAC", "ABAC"}},
{[]string{"RBAC", "ABAC"}, "v1.7.0-beta.0", []string{"RBAC", "ABAC"}},
{[]string{"RBAC"}, "v1.7.0", []string{"Node", "RBAC"}},
{[]string{"RBAC", "Webhook"}, "v1.7.0-beta.1", []string{"Node", "RBAC", "Webhook"}},
{[]string{"RBAC", "Webhook", "Node"}, "v1.7.0", []string{"RBAC", "Webhook", "Node"}},
{[]string{"Node", "RBAC", "Webhook"}, "v1.7.0", []string{"Node", "RBAC", "Webhook"}},
}
for _, rt := range tests {
k8sVersion, err := version.ParseSemantic(rt.version)
if err != nil {
t.Fatalf("couldn't parse given version")
}
actual := defaultAuthorizationModes(rt.authzModes, k8sVersion)
if strings.Join(actual, ",") != strings.Join(rt.expected, ",") {
t.Errorf(
"failed TestDefaultAuthorizationModes:\n\texpected: %s\n\t actual: %s",
strings.Join(rt.expected, ","),
strings.Join(actual, ","),
)
}
}
}

8
cmd/kubeadm/app/cmd/init.go
View File

@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/version"
)
var (
@ -253,13 +254,18 @@ func (i *Init) Run(out io.Writer) error {
// PHASE 5: Install and deploy all addons, and configure things as necessary
k8sVersion, err := version.ParseSemantic(i.cfg.KubernetesVersion)
if err != nil {
return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err)
}
// Create the necessary ServiceAccounts
err = apiconfigphase.CreateServiceAccounts(client)
if err != nil {
return err
}
err = apiconfigphase.CreateRBACRules(client)
err = apiconfigphase.CreateRBACRules(client, k8sVersion)
if err != nil {
return err
}

15
cmd/kubeadm/app/constants/constants.go
View File

@ -61,10 +61,11 @@ const (
// Some well-known users and groups in the core Kubernetes authorization system
ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
NodesClusterRoleBinding = "system:node"
// Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC
KubeDNSServiceAccountName = "kube-dns"
@ -89,9 +90,6 @@ const (
// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
MinExternalEtcdVersion = "3.0.14"
// DefaultAdmissionControl specifies the default admission control options that will be used
DefaultAdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds"
)
var (
@ -116,4 +114,7 @@ var (
// MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags
MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1")
// MinimumNodeAuthorizerVersion specifies the minimum kubernetes version that can be used for enabling the node authorizer
MinimumNodeAuthorizerVersion = version.MustParseSemantic("v1.7.0-beta.1")
)

1
cmd/kubeadm/app/master/BUILD
View File

@ -30,6 +30,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",

40
cmd/kubeadm/app/master/manifests.go
View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
api "k8s.io/client-go/pkg/api/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@ -43,6 +44,9 @@ import (
const (
DefaultCloudConfigPath = "/etc/kubernetes/cloud-config"
defaultv16AdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota"
defaultv17AdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota"
etcd = "etcd"
apiServer = "apiserver"
controllerManager = "controller-manager"
@ -326,7 +330,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
defaultArguments := map[string]string{
"insecure-port": "0",
"admission-control": kubeadmconstants.DefaultAdmissionControl,
"admission-control": defaultv16AdmissionControl,
"service-cluster-ip-range": cfg.Networking.ServiceSubnet,
"service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
"client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
@ -351,6 +355,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
}
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
// enable the NodeRestriction admission plugin
defaultArguments["admission-control"] = defaultv17AdmissionControl
}
command = getComponentBaseCommand(apiServer)
command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
@ -505,21 +513,25 @@ func getSelfHostedAPIServerEnv() []api.EnvVar {
func getAuthzParameters(modes []string) []string {
command := []string{}
// RBAC is always on. If the user specified
authzModes := []string{authzmodes.ModeRBAC}
for _, authzMode := range modes {
if len(authzMode) != 0 && authzMode != authzmodes.ModeRBAC {
authzModes = append(authzModes, authzMode)
}
switch authzMode {
case authzmodes.ModeABAC:
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
case authzmodes.ModeWebhook:
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}
strset := sets.NewString(modes...)
// RBAC must always be set, prepend that if not present in the list
// TODO(luxas): In v1.8, require the Node and RBAC authorizers to be present
// In v1.8 we can utilize the API defaulting/validation to do enforce this,
// currently this logic has to be split up into two places, which is sub-optimal.
// Cleanup work is possible in and should be done in v1.8
if !strset.Has(authzmodes.ModeRBAC) {
modes = append([]string{authzmodes.ModeRBAC}, modes...)
}
command = append(command, "--authorization-mode="+strings.Join(authzModes, ","))
if strset.Has(authzmodes.ModeABAC) {
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
}
if strset.Has(authzmodes.ModeWebhook) {
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}
command = append(command, "--authorization-mode="+strings.Join(modes, ","))
return command
}

18
cmd/kubeadm/app/master/manifests_test.go
View File

@ -522,7 +522,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -554,7 +554,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -587,7 +587,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -622,7 +622,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -884,7 +884,15 @@ func TestGetAuthzParameters(t *testing.T) {
{
authMode: []string{"ABAC", "RBAC", "Webhook"},
expected: []string{
"--authorization-mode=RBAC,ABAC,Webhook",
"--authorization-mode=ABAC,RBAC,Webhook",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},
},
{
authMode: []string{"Node", "RBAC", "Webhook", "ABAC"},
expected: []string{
"--authorization-mode=Node,RBAC,Webhook,ABAC",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},

1
cmd/kubeadm/app/phases/apiconfig/BUILD
View File

@ -20,6 +20,7 @@ go_library(
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",

40
cmd/kubeadm/app/phases/apiconfig/clusterroles.go
View File

@ -26,6 +26,7 @@ import (
rbac "k8s.io/client-go/pkg/apis/rbac/v1beta1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
"k8s.io/kubernetes/pkg/util/version"
)
const (
@ -74,7 +75,7 @@ func CreateServiceAccounts(clientset clientset.Interface) error {
}
// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(clientset *clientset.Clientset) error {
func CreateRBACRules(clientset *clientset.Clientset, k8sVersion *version.Version) error {
if err := createRoles(clientset); err != nil {
return err
}
@ -87,6 +88,9 @@ func CreateRBACRules(clientset *clientset.Clientset) error {
if err := createClusterRoleBindings(clientset); err != nil {
return err
}
if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil {
return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err)
}
fmt.Println("[apiconfig] Created RBAC rules")
return nil
@ -221,3 +225,37 @@ func createClusterRoleBindings(clientset *clientset.Clientset) error {
}
return nil
}
func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset *clientset.Clientset, k8sVersion *version.Version) error {
// If the server version is higher than the Node Authorizer's minimum, try to delete the Group=system:nodes->ClusterRole=system:node binding
// which is much more permissive than the Node Authorizer
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
// Nothing to do; the RoleBinding doesn't exist
return nil
}
return err
}
newSubjects := []rbac.Subject{}
for _, subject := range nodesRoleBinding.Subjects {
// Skip the subject that binds to the system:nodes group
if subject.Name == kubeadmconstants.NodesGroup && subject.Kind == "Group" {
continue
}
newSubjects = append(newSubjects, subject)
}
nodesRoleBinding.Subjects = newSubjects
if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil {
return err
}
}
return nil
}

1
cmd/kubeadm/app/phases/kubeconfig/BUILD
View File

@ -18,6 +18,7 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",

9
cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go
View File

@ -29,6 +29,7 @@ import (
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/pkg/util/node"
)
// BuildConfigProperties holds some simple information about how this phase should build the KubeConfig object
@ -54,9 +55,9 @@ type BuildConfigProperties struct {
// CreateInitKubeConfigFiles is called from the main init and does the work for the default phase behaviour
func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
hostname, err := os.Hostname()
if err != nil {
return err
nodeName := node.GetHostname("")
if len(nodeName) == 0 {
return fmt.Errorf("unable to get hostname for master node")
}
// Create a lightweight specification for what the files should look like
@ -69,7 +70,7 @@ func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
MakeClientCerts: true,
},
kubeadmconstants.KubeletKubeConfigFileName: {
ClientName: fmt.Sprintf("system:node:%s", hostname),
ClientName: fmt.Sprintf("system:node:%s", nodeName),
APIServer: masterEndpoint,
CertDir: pkiDir,
Organization: []string{kubeadmconstants.NodesGroup},