Merge pull request #46879 from luxas/kubeadm_enable_node_authorizer

Automatic merge from submit-queue

kubeadm: Enable the Node Authorizer/Admission plugin in v1.7

**What this PR does / why we need it**:

This is similar to https://github.com/kubernetes/kubernetes/pull/46796, but for kubeadm.
Basically it was a part of https://github.com/kubernetes/kubernetes/pull/46796, but there were some other upgradability and compability concerns for kubeadm I took care of while working today.

Example:

```console
$ kubeadm init --kubernetes-version v1.7.0-beta.0
[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.
[init] Using Kubernetes version: v1.7.0-beta.0
[init] Using Authorization mode: [RBAC Node]
...
$ sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf get secret foo
Error from server (Forbidden): User "system:node:thegopher" cannot get secrets in the namespace "default".: "no path found to object" (get secrets foo)

$ echo '{"apiVersion":"v1","kind":"Node","metadata":{"name":"foo"}}' | sudo kubectl create -f - --kubeconfig=/etc/kubernetes/kubelet.conf
Error from server (Forbidden): error when creating "STDIN": nodes "foo" is forbidden: node thegopher cannot modify node foo
```

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

Depends on https://github.com/kubernetes/kubernetes/pull/46864 (uses that PR as a base, will rebase once it's merged)

Please only review the second commit. Will also fix tests in a minute.

**Release note**:

```release-note
kubeadm: Enable the Node Authorizer/Admission plugin in v1.7 
```
@mikedanese @liggitt @pipejakob @roberthbailey @jbeda @timothysc
This commit is contained in:
Kubernetes Submit Queue 2017-06-08 05:26:58 -07:00 committed by GitHub
commit 2bcd3d1a01
12 changed files with 176 additions and 33 deletions

View File

@ -41,6 +41,7 @@ go_library(
"//cmd/kubeadm/app/util/token:go_default_library",
"//pkg/api:go_default_library",
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/printers:go_default_library",
"//pkg/util/i18n:go_default_library",
@ -54,6 +55,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
@ -65,6 +67,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
"defaults_test.go",
"reset_test.go",
"token_test.go",
],
@ -73,6 +76,7 @@ go_test(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//pkg/util/version:go_default_library",
],
)

View File

@ -21,10 +21,12 @@ import (
"net"
netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/util/version"
)
@ -54,8 +56,12 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
}
// Defaulting is made here because it's dependent on the version currently, which is determined above
// TODO(luxas): Cleanup this once we have dropped v1.6 support and move this code into the API group defaulting
cfg.AuthorizationModes = defaultAuthorizationModes(cfg.AuthorizationModes, k8sVersion)
fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes)
fmt.Printf("[init] Using Authorization modes: %v\n", cfg.AuthorizationModes)
// Warn about the limitations with the current cloudprovider solution.
if cfg.CloudProvider != "" {
@ -73,3 +79,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
return nil
}
func defaultAuthorizationModes(authzModes []string, k8sVersion *version.Version) []string {
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
strset := sets.NewString(authzModes...)
if !strset.Has(authzmodes.ModeNode) {
return append([]string{authzmodes.ModeNode}, authzModes...)
}
}
return authzModes
}

View File

@ -0,0 +1,54 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"strings"
"testing"
"k8s.io/kubernetes/pkg/util/version"
)
func TestDefaultAuthorizationModes(t *testing.T) {
var tests = []struct {
authzModes []string
version string
expected []string
}{
{[]string{"RBAC"}, "v1.6.0", []string{"RBAC"}},
{[]string{"RBAC", "ABAC"}, "v1.6.4", []string{"RBAC", "ABAC"}},
{[]string{"RBAC", "ABAC"}, "v1.7.0-beta.0", []string{"RBAC", "ABAC"}},
{[]string{"RBAC"}, "v1.7.0", []string{"Node", "RBAC"}},
{[]string{"RBAC", "Webhook"}, "v1.7.0-beta.1", []string{"Node", "RBAC", "Webhook"}},
{[]string{"RBAC", "Webhook", "Node"}, "v1.7.0", []string{"RBAC", "Webhook", "Node"}},
{[]string{"Node", "RBAC", "Webhook"}, "v1.7.0", []string{"Node", "RBAC", "Webhook"}},
}
for _, rt := range tests {
k8sVersion, err := version.ParseSemantic(rt.version)
if err != nil {
t.Fatalf("couldn't parse given version")
}
actual := defaultAuthorizationModes(rt.authzModes, k8sVersion)
if strings.Join(actual, ",") != strings.Join(rt.expected, ",") {
t.Errorf(
"failed TestDefaultAuthorizationModes:\n\texpected: %s\n\t actual: %s",
strings.Join(rt.expected, ","),
strings.Join(actual, ","),
)
}
}
}

View File

@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/version"
)
var (
@ -253,13 +254,18 @@ func (i *Init) Run(out io.Writer) error {
// PHASE 5: Install and deploy all addons, and configure things as necessary
k8sVersion, err := version.ParseSemantic(i.cfg.KubernetesVersion)
if err != nil {
return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err)
}
// Create the necessary ServiceAccounts
err = apiconfigphase.CreateServiceAccounts(client)
if err != nil {
return err
}
err = apiconfigphase.CreateRBACRules(client)
err = apiconfigphase.CreateRBACRules(client, k8sVersion)
if err != nil {
return err
}

View File

@ -61,10 +61,11 @@ const (
// Some well-known users and groups in the core Kubernetes authorization system
ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
ControllerManagerUser = "system:kube-controller-manager"
SchedulerUser = "system:kube-scheduler"
MastersGroup = "system:masters"
NodesGroup = "system:nodes"
NodesClusterRoleBinding = "system:node"
// Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC
KubeDNSServiceAccountName = "kube-dns"
@ -89,9 +90,6 @@ const (
// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
MinExternalEtcdVersion = "3.0.14"
// DefaultAdmissionControl specifies the default admission control options that will be used
DefaultAdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds"
)
var (
@ -116,4 +114,7 @@ var (
// MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags
MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1")
// MinimumNodeAuthorizerVersion specifies the minimum kubernetes version that can be used for enabling the node authorizer
MinimumNodeAuthorizerVersion = version.MustParseSemantic("v1.7.0-beta.1")
)

View File

@ -30,6 +30,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",

View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
api "k8s.io/client-go/pkg/api/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@ -43,6 +44,9 @@ import (
const (
DefaultCloudConfigPath = "/etc/kubernetes/cloud-config"
defaultv16AdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota"
defaultv17AdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota"
etcd = "etcd"
apiServer = "apiserver"
controllerManager = "controller-manager"
@ -326,7 +330,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
defaultArguments := map[string]string{
"insecure-port": "0",
"admission-control": kubeadmconstants.DefaultAdmissionControl,
"admission-control": defaultv16AdmissionControl,
"service-cluster-ip-range": cfg.Networking.ServiceSubnet,
"service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
"client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
@ -351,6 +355,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
}
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
// enable the NodeRestriction admission plugin
defaultArguments["admission-control"] = defaultv17AdmissionControl
}
command = getComponentBaseCommand(apiServer)
command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
@ -505,21 +513,25 @@ func getSelfHostedAPIServerEnv() []api.EnvVar {
func getAuthzParameters(modes []string) []string {
command := []string{}
// RBAC is always on. If the user specified
authzModes := []string{authzmodes.ModeRBAC}
for _, authzMode := range modes {
if len(authzMode) != 0 && authzMode != authzmodes.ModeRBAC {
authzModes = append(authzModes, authzMode)
}
switch authzMode {
case authzmodes.ModeABAC:
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
case authzmodes.ModeWebhook:
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}
strset := sets.NewString(modes...)
// RBAC must always be set, prepend that if not present in the list
// TODO(luxas): In v1.8, require the Node and RBAC authorizers to be present
// In v1.8 we can utilize the API defaulting/validation to do enforce this,
// currently this logic has to be split up into two places, which is sub-optimal.
// Cleanup work is possible in and should be done in v1.8
if !strset.Has(authzmodes.ModeRBAC) {
modes = append([]string{authzmodes.ModeRBAC}, modes...)
}
command = append(command, "--authorization-mode="+strings.Join(authzModes, ","))
if strset.Has(authzmodes.ModeABAC) {
command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
}
if strset.Has(authzmodes.ModeWebhook) {
command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
}
command = append(command, "--authorization-mode="+strings.Join(modes, ","))
return command
}

View File

@ -522,7 +522,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -554,7 +554,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -587,7 +587,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -622,7 +622,7 @@ func TestGetAPIServerCommand(t *testing.T) {
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@ -884,7 +884,15 @@ func TestGetAuthzParameters(t *testing.T) {
{
authMode: []string{"ABAC", "RBAC", "Webhook"},
expected: []string{
"--authorization-mode=RBAC,ABAC,Webhook",
"--authorization-mode=ABAC,RBAC,Webhook",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},
},
{
authMode: []string{"Node", "RBAC", "Webhook", "ABAC"},
expected: []string{
"--authorization-mode=Node,RBAC,Webhook,ABAC",
"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
},

View File

@ -20,6 +20,7 @@ go_library(
"//pkg/bootstrap/api:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",

View File

@ -26,6 +26,7 @@ import (
rbac "k8s.io/client-go/pkg/apis/rbac/v1beta1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
"k8s.io/kubernetes/pkg/util/version"
)
const (
@ -74,7 +75,7 @@ func CreateServiceAccounts(clientset clientset.Interface) error {
}
// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(clientset *clientset.Clientset) error {
func CreateRBACRules(clientset *clientset.Clientset, k8sVersion *version.Version) error {
if err := createRoles(clientset); err != nil {
return err
}
@ -87,6 +88,9 @@ func CreateRBACRules(clientset *clientset.Clientset) error {
if err := createClusterRoleBindings(clientset); err != nil {
return err
}
if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil {
return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err)
}
fmt.Println("[apiconfig] Created RBAC rules")
return nil
@ -245,3 +249,37 @@ func createClusterRoleBindings(clientset *clientset.Clientset) error {
}
return nil
}
func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset *clientset.Clientset, k8sVersion *version.Version) error {
// If the server version is higher than the Node Authorizer's minimum, try to delete the Group=system:nodes->ClusterRole=system:node binding
// which is much more permissive than the Node Authorizer
if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
// Nothing to do; the RoleBinding doesn't exist
return nil
}
return err
}
newSubjects := []rbac.Subject{}
for _, subject := range nodesRoleBinding.Subjects {
// Skip the subject that binds to the system:nodes group
if subject.Name == kubeadmconstants.NodesGroup && subject.Kind == "Group" {
continue
}
newSubjects = append(newSubjects, subject)
}
nodesRoleBinding.Subjects = newSubjects
if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil {
return err
}
}
return nil
}

View File

@ -18,6 +18,7 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",

View File

@ -29,6 +29,7 @@ import (
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/pkg/util/node"
)
// BuildConfigProperties holds some simple information about how this phase should build the KubeConfig object
@ -54,9 +55,9 @@ type BuildConfigProperties struct {
// CreateInitKubeConfigFiles is called from the main init and does the work for the default phase behaviour
func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
hostname, err := os.Hostname()
if err != nil {
return err
nodeName := node.GetHostname("")
if len(nodeName) == 0 {
return fmt.Errorf("unable to get hostname for master node")
}
// Create a lightweight specification for what the files should look like
@ -69,7 +70,7 @@ func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
MakeClientCerts: true,
},
kubeadmconstants.KubeletKubeConfigFileName: {
ClientName: fmt.Sprintf("system:node:%s", hostname),
ClientName: fmt.Sprintf("system:node:%s", nodeName),
APIServer: masterEndpoint,
CertDir: pkiDir,
Organization: []string{kubeadmconstants.NodesGroup},