rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-08-19 07:17:30 +00:00
Code Issues Releases Wiki Activity

update k8s defaults, CIS recommendations

Browse Source
This commit is contained in:
Jason Greathouse 2019-01-07 13:52:57 -06:00 committed by Alena Prokharchyk
parent 2bf2cd8f5b
commit 7afa6e927e
6 changed files with 127 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,3 +7,4 @@
/trash.lock /trash.lock
kube_config* kube_config*
/rke /rke
.vscode

2
cluster/cluster.go
View File

@ -37,6 +37,7 @@ type Cluster struct {
ClusterDomain string ClusterDomain string
ClusterCIDR string ClusterCIDR string
ClusterDNSServer string ClusterDNSServer string
DinD bool
DockerDialerFactory hosts.DialerFactory DockerDialerFactory hosts.DialerFactory
EtcdHosts []*hosts.Host EtcdHosts []*hosts.Host
EtcdReadyHosts []*hosts.Host EtcdReadyHosts []*hosts.Host
@ -150,6 +151,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
RancherKubernetesEngineConfig: *rkeConfig, RancherKubernetesEngineConfig: *rkeConfig,
ConfigPath: flags.ClusterFilePath, ConfigPath: flags.ClusterFilePath,
ConfigDir: flags.ConfigDir, ConfigDir: flags.ConfigDir,
DinD: flags.DinD,
StateFilePath: GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir), StateFilePath: GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir),
PrivateRegistriesMap: make(map[string]v3.PrivateRegistry), PrivateRegistriesMap: make(map[string]v3.PrivateRegistry),
} }

1
cluster/defaults.go
View File

@ -54,6 +54,7 @@ const (
type ExternalFlags struct { type ExternalFlags struct {
ConfigDir string ConfigDir string
ClusterFilePath string ClusterFilePath string
DinD bool
DisablePortCheck bool DisablePortCheck bool
Local bool Local bool
UpdateOnly bool UpdateOnly bool

163
cluster/plan.go
View File

@ -117,36 +117,48 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
c.getRKEToolsEntryPoint(), c.getRKEToolsEntryPoint(),
"kube-apiserver", "kube-apiserver",
} }
baseEnabledAdmissionPlugins := []string{
"DefaultStorageClass",
"DefaultTolerationSeconds",
"LimitRanger",
"NamespaceLifecycle",
"NodeRestriction",
"PersistentVolumeLabel",
"ResourceQuota",
"ServiceAccount",
}
CommandArgs := map[string]string{ CommandArgs := map[string]string{
"insecure-bind-address": "127.0.0.1",
"bind-address": "0.0.0.0",
"insecure-port": "0",
"secure-port": "6443",
"cloud-provider": c.CloudProvider.Name,
"allow-privileged": "true", "allow-privileged": "true",
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", "anonymous-auth": "false",
"service-cluster-ip-range": c.Services.KubeAPI.ServiceClusterIPRange, "bind-address": "0.0.0.0",
"service-node-port-range": c.Services.KubeAPI.ServiceNodePortRange,
"storage-backend": "etcd3",
"client-ca-file": pki.GetCertPath(pki.CACertName), "client-ca-file": pki.GetCertPath(pki.CACertName),
"tls-cert-file": pki.GetCertPath(pki.KubeAPICertName), "cloud-provider": c.CloudProvider.Name,
"tls-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
"kubelet-client-certificate": pki.GetCertPath(pki.KubeAPICertName),
"kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName),
"service-account-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
"etcd-cafile": etcdCAClientCert, "etcd-cafile": etcdCAClientCert,
"etcd-certfile": etcdClientCert, "etcd-certfile": etcdClientCert,
"etcd-keyfile": etcdClientKey, "etcd-keyfile": etcdClientKey,
"etcd-servers": etcdConnectionString,
"etcd-prefix": etcdPathPrefix, "etcd-prefix": etcdPathPrefix,
"requestheader-client-ca-file": pki.GetCertPath(pki.RequestHeaderCACertName), "etcd-servers": etcdConnectionString,
"requestheader-allowed-names": pki.APIProxyClientCertName, "insecure-port": "0",
"proxy-client-key-file": pki.GetKeyPath(pki.APIProxyClientCertName), "kubelet-client-certificate": pki.GetCertPath(pki.KubeAPICertName),
"kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName),
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
"profiling": "false",
"proxy-client-cert-file": pki.GetCertPath(pki.APIProxyClientCertName), "proxy-client-cert-file": pki.GetCertPath(pki.APIProxyClientCertName),
"proxy-client-key-file": pki.GetKeyPath(pki.APIProxyClientCertName),
"requestheader-allowed-names": pki.APIProxyClientCertName,
"requestheader-client-ca-file": pki.GetCertPath(pki.RequestHeaderCACertName),
"requestheader-extra-headers-prefix": "X-Remote-Extra-", "requestheader-extra-headers-prefix": "X-Remote-Extra-",
"requestheader-group-headers": "X-Remote-Group", "requestheader-group-headers": "X-Remote-Group",
"requestheader-username-headers": "X-Remote-User", "requestheader-username-headers": "X-Remote-User",
"repair-malformed-updates": "false",
"secure-port": "6443",
"service-account-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
"service-account-lookup": "true",
"service-cluster-ip-range": c.Services.KubeAPI.ServiceClusterIPRange,
"service-node-port-range": c.Services.KubeAPI.ServiceNodePortRange,
"storage-backend": "etcd3",
"tls-cert-file": pki.GetCertPath(pki.KubeAPICertName),
"tls-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
} }
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
CommandArgs["cloud-config"] = cloudConfigFileName CommandArgs["cloud-config"] = cloudConfigFileName
@ -180,6 +192,28 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
if c.Authorization.Mode == services.RBACAuthorizationMode { if c.Authorization.Mode == services.RBACAuthorizationMode {
CommandArgs["authorization-mode"] = "Node,RBAC" CommandArgs["authorization-mode"] = "Node,RBAC"
} }
// PodSecurityPolicy
if c.Services.KubeAPI.PodSecurityPolicy {
CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
baseEnabledAdmissionPlugins = append(baseEnabledAdmissionPlugins, "PodSecurityPolicy")
}
// Admission control plugins
// Resolution order:
// k8s_defaults.go K8sVersionServiceOptions
// enabledAdmissionPlugins
// cluster.yml extra_args overwrites it all
for _, optionName := range admissionControlOptionNames {
if _, ok := CommandArgs[optionName]; ok {
enabledAdmissionPlugins := strings.Split(CommandArgs[optionName], ",")
enabledAdmissionPlugins = append(enabledAdmissionPlugins, baseEnabledAdmissionPlugins...)
// Join unique slice as arg
CommandArgs[optionName] = strings.Join(util.UniqueStringSlice(enabledAdmissionPlugins), ",")
break
}
}
if c.Services.KubeAPI.PodSecurityPolicy { if c.Services.KubeAPI.PodSecurityPolicy {
CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true" CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
for _, optionName := range admissionControlOptionNames { for _, optionName := range admissionControlOptionNames {
@ -240,21 +274,27 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
} }
CommandArgs := map[string]string{ CommandArgs := map[string]string{
"address": "0.0.0.0", "address": "127.0.0.1",
"cloud-provider": c.CloudProvider.Name,
"allow-untagged-cloud": "true", "allow-untagged-cloud": "true",
"allocate-node-cidrs": "true",
"cloud-provider": c.CloudProvider.Name,
"cluster-cidr": c.ClusterCIDR,
"configure-cloud-routes": "false", "configure-cloud-routes": "false",
"leader-elect": "true",
"kubeconfig": pki.GetConfigPath(pki.KubeControllerCertName),
"enable-hostpath-provisioner": "false", "enable-hostpath-provisioner": "false",
"kubeconfig": pki.GetConfigPath(pki.KubeControllerCertName),
"leader-elect": "true",
"node-monitor-grace-period": "40s", "node-monitor-grace-period": "40s",
"pod-eviction-timeout": "5m0s", "pod-eviction-timeout": "5m0s",
"v": "2", "profiling": "false",
"allocate-node-cidrs": "true",
"cluster-cidr": c.ClusterCIDR,
"service-cluster-ip-range": c.Services.KubeController.ServiceClusterIPRange,
"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
"root-ca-file": pki.GetCertPath(pki.CACertName), "root-ca-file": pki.GetCertPath(pki.CACertName),
"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
"service-cluster-ip-range": c.Services.KubeController.ServiceClusterIPRange,
"terminated-pod-gc-threshold": "1000",
"v": "2",
}
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
if c.DinD {
CommandArgs["address"] = "0.0.0.0"
} }
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
CommandArgs["cloud-config"] = cloudConfigFileName CommandArgs["cloud-config"] = cloudConfigFileName
@ -332,29 +372,31 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
} }
CommandArgs := map[string]string{ CommandArgs := map[string]string{
"v": "2", "address": "0.0.0.0",
"address": "0.0.0.0", "allow-privileged": "true",
"cadvisor-port": "0", //depricated in 1.12 "anonymous-auth": "false",
"read-only-port": "0", "authentication-token-webhook": "true",
"cluster-domain": c.ClusterDomain, "cgroups-per-qos": "True",
"pod-infra-container-image": c.Services.Kubelet.InfraContainerImage, "client-ca-file": pki.GetCertPath(pki.CACertName),
"cgroups-per-qos": "True", "cloud-provider": c.CloudProvider.Name,
"enforce-node-allocatable": "", "cluster-dns": c.ClusterDNSServer,
"hostname-override": host.HostnameOverride, "cluster-domain": c.ClusterDomain,
"cluster-dns": c.ClusterDNSServer, "cni-bin-dir": "/opt/cni/bin",
"network-plugin": "cni", "cni-conf-dir": "/etc/cni/net.d",
"cni-conf-dir": "/etc/cni/net.d", "enforce-node-allocatable": "",
"cni-bin-dir": "/opt/cni/bin", "event-qps": "0",
"resolv-conf": "/etc/resolv.conf", "fail-swap-on": strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
"allow-privileged": "true", "hostname-override": host.HostnameOverride,
"cloud-provider": c.CloudProvider.Name, "kubeconfig": pki.GetConfigPath(pki.KubeNodeCertName),
"kubeconfig": pki.GetConfigPath(pki.KubeNodeCertName), "make-iptables-util-chains": "true",
"client-ca-file": pki.GetCertPath(pki.CACertName), "network-plugin": "cni",
"anonymous-auth": "false", "pod-infra-container-image": c.Services.Kubelet.InfraContainerImage,
"volume-plugin-dir": "/var/lib/kubelet/volumeplugins", "read-only-port": "0",
"fail-swap-on": strconv.FormatBool(c.Services.Kubelet.FailSwapOn), "resolv-conf": "/etc/resolv.conf",
"root-dir": path.Join(prefixPath, "/var/lib/kubelet"), "root-dir": path.Join(prefixPath, "/var/lib/kubelet"),
"authentication-token-webhook": "true", "streaming-connection-idle-timeout": "30m",
"volume-plugin-dir": "/var/lib/kubelet/volumeplugins",
"v": "2",
} }
if host.IsControl && !host.IsWorker { if host.IsControl && !host.IsWorker {
CommandArgs["register-with-taints"] = unschedulableControlTaint CommandArgs["register-with-taints"] = unschedulableControlTaint
@ -371,11 +413,11 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile))) fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile)))
} }
if len(c.PrivateRegistriesMap) > 0 { if len(c.PrivateRegistriesMap) > 0 {
kubeletDcokerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap) kubeletDockerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap)
c.Services.Kubelet.ExtraEnv = append( c.Services.Kubelet.ExtraEnv = append(
c.Services.Kubelet.ExtraEnv, c.Services.Kubelet.ExtraEnv,
fmt.Sprintf("%s=%s", KubeletDockerConfigEnv, fmt.Sprintf("%s=%s", KubeletDockerConfigEnv,
b64.StdEncoding.EncodeToString([]byte(kubeletDcokerConfig)))) b64.StdEncoding.EncodeToString([]byte(kubeletDockerConfig))))
c.Services.Kubelet.ExtraEnv = append( c.Services.Kubelet.ExtraEnv = append(
c.Services.Kubelet.ExtraEnv, c.Services.Kubelet.ExtraEnv,
@ -468,11 +510,14 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string) v3.
CommandArgs := map[string]string{ CommandArgs := map[string]string{
"cluster-cidr": c.ClusterCIDR, "cluster-cidr": c.ClusterCIDR,
"v": "2", "v": "2",
"healthz-bind-address": "0.0.0.0", "healthz-bind-address": "127.0.0.1",
"hostname-override": host.HostnameOverride, "hostname-override": host.HostnameOverride,
"kubeconfig": pki.GetConfigPath(pki.KubeProxyCertName), "kubeconfig": pki.GetConfigPath(pki.KubeProxyCertName),
} }
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
if c.DinD {
CommandArgs["healthz-bind-address"] = "0.0.0.0"
}
// check if our version has specific options for this component // check if our version has specific options for this component
serviceOptions := c.GetKubernetesServicesOptions() serviceOptions := c.GetKubernetesServicesOptions()
if serviceOptions.Kubeproxy != nil { if serviceOptions.Kubeproxy != nil {
@ -566,10 +611,16 @@ func (c *Cluster) BuildSchedulerProcess(prefixPath string) v3.Process {
CommandArgs := map[string]string{ CommandArgs := map[string]string{
"leader-elect": "true", "leader-elect": "true",
"v": "2", "v": "2",
"address": "0.0.0.0", "address": "127.0.0.1",
"profiling": "false",
"kubeconfig": pki.GetConfigPath(pki.KubeSchedulerCertName), "kubeconfig": pki.GetConfigPath(pki.KubeSchedulerCertName),
} }
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
if c.DinD {
CommandArgs["address"] = "0.0.0.0"
}
// check if our version has specific options for this component // check if our version has specific options for this component
serviceOptions := c.GetKubernetesServicesOptions() serviceOptions := c.GetKubernetesServicesOptions()
if serviceOptions.Scheduler != nil { if serviceOptions.Scheduler != nil {

1
cmd/up.go
View File

@ -315,6 +315,7 @@ func clusterUpDind(ctx *cli.Context) error {
dialers := hosts.GetDialerOptions(hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil) dialers := hosts.GetDialerOptions(hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil)
// setting up flags // setting up flags
flags := cluster.GetExternalFlags(false, false, disablePortCheck, "", filePath) flags := cluster.GetExternalFlags(false, false, disablePortCheck, "", filePath)
flags.DinD = true
if ctx.Bool("init") { if ctx.Bool("init") {
return ClusterInit(context.Background(), rkeConfig, dialers, flags) return ClusterInit(context.Background(), rkeConfig, dialers, flags)

14
util/util.go
View File

@ -37,3 +37,17 @@ func ErrList(e []error) error {
} }
return nil return nil
} }
// UniqueStringSlice - Input slice, retrun slice with unique elements. Will not maintain order.
func UniqueStringSlice(elements []string) []string {
encountered := map[string]bool{}
result := []string{}
for v := range elements {
if !encountered[elements[v]] {
encountered[elements[v]] = true
result = append(result, elements[v])
}
}
return result
}