mirror of
https://github.com/rancher/rke.git
synced 2025-04-27 19:25:44 +00:00
update k8s defaults, CIS recommendations
This commit is contained in:
Jason Greathouse
Alena Prokharchyk
parent
2bf2cd8f5b
commit
7afa6e927e
3
.gitignore
vendored
3
.gitignore
vendored
@ -6,4 +6,5 @@
|
||||
/.trash-cache
|
||||
/trash.lock
|
||||
kube_config*
|
||||
/rke
|
||||
/rke
|
||||
.vscode
|
||||
|
||||
@ -37,6 +37,7 @@ type Cluster struct {
|
||||
ClusterDomain string
|
||||
ClusterCIDR string
|
||||
ClusterDNSServer string
|
||||
DinD bool
|
||||
DockerDialerFactory hosts.DialerFactory
|
||||
EtcdHosts []*hosts.Host
|
||||
EtcdReadyHosts []*hosts.Host
|
||||
@ -150,6 +151,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
|
||||
RancherKubernetesEngineConfig: *rkeConfig,
|
||||
ConfigPath: flags.ClusterFilePath,
|
||||
ConfigDir: flags.ConfigDir,
|
||||
DinD: flags.DinD,
|
||||
StateFilePath: GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir),
|
||||
PrivateRegistriesMap: make(map[string]v3.PrivateRegistry),
|
||||
}
|
||||
|
||||
@ -54,6 +54,7 @@ const (
|
||||
type ExternalFlags struct {
|
||||
ConfigDir string
|
||||
ClusterFilePath string
|
||||
DinD bool
|
||||
DisablePortCheck bool
|
||||
Local bool
|
||||
UpdateOnly bool
|
||||
|
||||
163
cluster/plan.go
163
cluster/plan.go
@ -117,36 +117,48 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
|
||||
c.getRKEToolsEntryPoint(),
|
||||
"kube-apiserver",
|
||||
}
|
||||
|
||||
baseEnabledAdmissionPlugins := []string{
|
||||
"DefaultStorageClass",
|
||||
"DefaultTolerationSeconds",
|
||||
"LimitRanger",
|
||||
"NamespaceLifecycle",
|
||||
"NodeRestriction",
|
||||
"PersistentVolumeLabel",
|
||||
"ResourceQuota",
|
||||
"ServiceAccount",
|
||||
}
|
||||
CommandArgs := map[string]string{
|
||||
"insecure-bind-address": "127.0.0.1",
|
||||
"bind-address": "0.0.0.0",
|
||||
"insecure-port": "0",
|
||||
"secure-port": "6443",
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"allow-privileged": "true",
|
||||
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
|
||||
"service-cluster-ip-range": c.Services.KubeAPI.ServiceClusterIPRange,
|
||||
"service-node-port-range": c.Services.KubeAPI.ServiceNodePortRange,
|
||||
"storage-backend": "etcd3",
|
||||
"anonymous-auth": "false",
|
||||
"bind-address": "0.0.0.0",
|
||||
"client-ca-file": pki.GetCertPath(pki.CACertName),
|
||||
"tls-cert-file": pki.GetCertPath(pki.KubeAPICertName),
|
||||
"tls-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
|
||||
"kubelet-client-certificate": pki.GetCertPath(pki.KubeAPICertName),
|
||||
"kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName),
|
||||
"service-account-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"etcd-cafile": etcdCAClientCert,
|
||||
"etcd-certfile": etcdClientCert,
|
||||
"etcd-keyfile": etcdClientKey,
|
||||
"etcd-servers": etcdConnectionString,
|
||||
"etcd-prefix": etcdPathPrefix,
|
||||
"requestheader-client-ca-file": pki.GetCertPath(pki.RequestHeaderCACertName),
|
||||
"requestheader-allowed-names": pki.APIProxyClientCertName,
|
||||
"proxy-client-key-file": pki.GetKeyPath(pki.APIProxyClientCertName),
|
||||
"etcd-servers": etcdConnectionString,
|
||||
"insecure-port": "0",
|
||||
"kubelet-client-certificate": pki.GetCertPath(pki.KubeAPICertName),
|
||||
"kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName),
|
||||
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
|
||||
"profiling": "false",
|
||||
"proxy-client-cert-file": pki.GetCertPath(pki.APIProxyClientCertName),
|
||||
"proxy-client-key-file": pki.GetKeyPath(pki.APIProxyClientCertName),
|
||||
"requestheader-allowed-names": pki.APIProxyClientCertName,
|
||||
"requestheader-client-ca-file": pki.GetCertPath(pki.RequestHeaderCACertName),
|
||||
"requestheader-extra-headers-prefix": "X-Remote-Extra-",
|
||||
"requestheader-group-headers": "X-Remote-Group",
|
||||
"requestheader-username-headers": "X-Remote-User",
|
||||
"repair-malformed-updates": "false",
|
||||
"secure-port": "6443",
|
||||
"service-account-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
|
||||
"service-account-lookup": "true",
|
||||
"service-cluster-ip-range": c.Services.KubeAPI.ServiceClusterIPRange,
|
||||
"service-node-port-range": c.Services.KubeAPI.ServiceNodePortRange,
|
||||
"storage-backend": "etcd3",
|
||||
"tls-cert-file": pki.GetCertPath(pki.KubeAPICertName),
|
||||
"tls-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
|
||||
}
|
||||
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
|
||||
CommandArgs["cloud-config"] = cloudConfigFileName
|
||||
@ -180,6 +192,28 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
|
||||
if c.Authorization.Mode == services.RBACAuthorizationMode {
|
||||
CommandArgs["authorization-mode"] = "Node,RBAC"
|
||||
}
|
||||
|
||||
// PodSecurityPolicy
|
||||
if c.Services.KubeAPI.PodSecurityPolicy {
|
||||
CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
|
||||
baseEnabledAdmissionPlugins = append(baseEnabledAdmissionPlugins, "PodSecurityPolicy")
|
||||
}
|
||||
|
||||
// Admission control plugins
|
||||
// Resolution order:
|
||||
// k8s_defaults.go K8sVersionServiceOptions
|
||||
// enabledAdmissionPlugins
|
||||
// cluster.yml extra_args overwrites it all
|
||||
for _, optionName := range admissionControlOptionNames {
|
||||
if _, ok := CommandArgs[optionName]; ok {
|
||||
enabledAdmissionPlugins := strings.Split(CommandArgs[optionName], ",")
|
||||
enabledAdmissionPlugins = append(enabledAdmissionPlugins, baseEnabledAdmissionPlugins...)
|
||||
|
||||
// Join unique slice as arg
|
||||
CommandArgs[optionName] = strings.Join(util.UniqueStringSlice(enabledAdmissionPlugins), ",")
|
||||
break
|
||||
}
|
||||
}
|
||||
if c.Services.KubeAPI.PodSecurityPolicy {
|
||||
CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
|
||||
for _, optionName := range admissionControlOptionNames {
|
||||
@ -240,21 +274,27 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
|
||||
}
|
||||
|
||||
CommandArgs := map[string]string{
|
||||
"address": "0.0.0.0",
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"address": "127.0.0.1",
|
||||
"allow-untagged-cloud": "true",
|
||||
"allocate-node-cidrs": "true",
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"cluster-cidr": c.ClusterCIDR,
|
||||
"configure-cloud-routes": "false",
|
||||
"leader-elect": "true",
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeControllerCertName),
|
||||
"enable-hostpath-provisioner": "false",
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeControllerCertName),
|
||||
"leader-elect": "true",
|
||||
"node-monitor-grace-period": "40s",
|
||||
"pod-eviction-timeout": "5m0s",
|
||||
"v": "2",
|
||||
"allocate-node-cidrs": "true",
|
||||
"cluster-cidr": c.ClusterCIDR,
|
||||
"service-cluster-ip-range": c.Services.KubeController.ServiceClusterIPRange,
|
||||
"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
|
||||
"profiling": "false",
|
||||
"root-ca-file": pki.GetCertPath(pki.CACertName),
|
||||
"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
|
||||
"service-cluster-ip-range": c.Services.KubeController.ServiceClusterIPRange,
|
||||
"terminated-pod-gc-threshold": "1000",
|
||||
"v": "2",
|
||||
}
|
||||
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
|
||||
if c.DinD {
|
||||
CommandArgs["address"] = "0.0.0.0"
|
||||
}
|
||||
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
|
||||
CommandArgs["cloud-config"] = cloudConfigFileName
|
||||
@ -332,29 +372,31 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
|
||||
}
|
||||
|
||||
CommandArgs := map[string]string{
|
||||
"v": "2",
|
||||
"address": "0.0.0.0",
|
||||
"cadvisor-port": "0", //depricated in 1.12
|
||||
"read-only-port": "0",
|
||||
"cluster-domain": c.ClusterDomain,
|
||||
"pod-infra-container-image": c.Services.Kubelet.InfraContainerImage,
|
||||
"cgroups-per-qos": "True",
|
||||
"enforce-node-allocatable": "",
|
||||
"hostname-override": host.HostnameOverride,
|
||||
"cluster-dns": c.ClusterDNSServer,
|
||||
"network-plugin": "cni",
|
||||
"cni-conf-dir": "/etc/cni/net.d",
|
||||
"cni-bin-dir": "/opt/cni/bin",
|
||||
"resolv-conf": "/etc/resolv.conf",
|
||||
"allow-privileged": "true",
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeNodeCertName),
|
||||
"client-ca-file": pki.GetCertPath(pki.CACertName),
|
||||
"anonymous-auth": "false",
|
||||
"volume-plugin-dir": "/var/lib/kubelet/volumeplugins",
|
||||
"fail-swap-on": strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
|
||||
"root-dir": path.Join(prefixPath, "/var/lib/kubelet"),
|
||||
"authentication-token-webhook": "true",
|
||||
"address": "0.0.0.0",
|
||||
"allow-privileged": "true",
|
||||
"anonymous-auth": "false",
|
||||
"authentication-token-webhook": "true",
|
||||
"cgroups-per-qos": "True",
|
||||
"client-ca-file": pki.GetCertPath(pki.CACertName),
|
||||
"cloud-provider": c.CloudProvider.Name,
|
||||
"cluster-dns": c.ClusterDNSServer,
|
||||
"cluster-domain": c.ClusterDomain,
|
||||
"cni-bin-dir": "/opt/cni/bin",
|
||||
"cni-conf-dir": "/etc/cni/net.d",
|
||||
"enforce-node-allocatable": "",
|
||||
"event-qps": "0",
|
||||
"fail-swap-on": strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
|
||||
"hostname-override": host.HostnameOverride,
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeNodeCertName),
|
||||
"make-iptables-util-chains": "true",
|
||||
"network-plugin": "cni",
|
||||
"pod-infra-container-image": c.Services.Kubelet.InfraContainerImage,
|
||||
"read-only-port": "0",
|
||||
"resolv-conf": "/etc/resolv.conf",
|
||||
"root-dir": path.Join(prefixPath, "/var/lib/kubelet"),
|
||||
"streaming-connection-idle-timeout": "30m",
|
||||
"volume-plugin-dir": "/var/lib/kubelet/volumeplugins",
|
||||
"v": "2",
|
||||
}
|
||||
if host.IsControl && !host.IsWorker {
|
||||
CommandArgs["register-with-taints"] = unschedulableControlTaint
|
||||
@ -371,11 +413,11 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
|
||||
fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile)))
|
||||
}
|
||||
if len(c.PrivateRegistriesMap) > 0 {
|
||||
kubeletDcokerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap)
|
||||
kubeletDockerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap)
|
||||
c.Services.Kubelet.ExtraEnv = append(
|
||||
c.Services.Kubelet.ExtraEnv,
|
||||
fmt.Sprintf("%s=%s", KubeletDockerConfigEnv,
|
||||
b64.StdEncoding.EncodeToString([]byte(kubeletDcokerConfig))))
|
||||
b64.StdEncoding.EncodeToString([]byte(kubeletDockerConfig))))
|
||||
|
||||
c.Services.Kubelet.ExtraEnv = append(
|
||||
c.Services.Kubelet.ExtraEnv,
|
||||
@ -468,11 +510,14 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string) v3.
|
||||
CommandArgs := map[string]string{
|
||||
"cluster-cidr": c.ClusterCIDR,
|
||||
"v": "2",
|
||||
"healthz-bind-address": "0.0.0.0",
|
||||
"healthz-bind-address": "127.0.0.1",
|
||||
"hostname-override": host.HostnameOverride,
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeProxyCertName),
|
||||
}
|
||||
|
||||
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
|
||||
if c.DinD {
|
||||
CommandArgs["healthz-bind-address"] = "0.0.0.0"
|
||||
}
|
||||
// check if our version has specific options for this component
|
||||
serviceOptions := c.GetKubernetesServicesOptions()
|
||||
if serviceOptions.Kubeproxy != nil {
|
||||
@ -566,10 +611,16 @@ func (c *Cluster) BuildSchedulerProcess(prefixPath string) v3.Process {
|
||||
CommandArgs := map[string]string{
|
||||
"leader-elect": "true",
|
||||
"v": "2",
|
||||
"address": "0.0.0.0",
|
||||
"address": "127.0.0.1",
|
||||
"profiling": "false",
|
||||
"kubeconfig": pki.GetConfigPath(pki.KubeSchedulerCertName),
|
||||
}
|
||||
|
||||
// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
|
||||
if c.DinD {
|
||||
CommandArgs["address"] = "0.0.0.0"
|
||||
}
|
||||
|
||||
// check if our version has specific options for this component
|
||||
serviceOptions := c.GetKubernetesServicesOptions()
|
||||
if serviceOptions.Scheduler != nil {
|
||||
|
||||
@ -315,6 +315,7 @@ func clusterUpDind(ctx *cli.Context) error {
|
||||
dialers := hosts.GetDialerOptions(hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil)
|
||||
// setting up flags
|
||||
flags := cluster.GetExternalFlags(false, false, disablePortCheck, "", filePath)
|
||||
flags.DinD = true
|
||||
|
||||
if ctx.Bool("init") {
|
||||
return ClusterInit(context.Background(), rkeConfig, dialers, flags)
|
||||
|
||||
14
util/util.go
14
util/util.go
@ -37,3 +37,17 @@ func ErrList(e []error) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UniqueStringSlice - Input slice, retrun slice with unique elements. Will not maintain order.
|
||||
func UniqueStringSlice(elements []string) []string {
|
||||
encountered := map[string]bool{}
|
||||
result := []string{}
|
||||
|
||||
for v := range elements {
|
||||
if !encountered[elements[v]] {
|
||||
encountered[elements[v]] = true
|
||||
result = append(result, elements[v])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user