update k8s defaults, CIS recommendations

.gitignore

@@ -7,3 +7,4 @@
 /trash.lock
 kube_config*
 /rke
+.vscode

cluster/cluster.go

@@ -37,6 +37,7 @@ type Cluster struct {
 	ClusterDomain                    string
 	ClusterCIDR                      string
 	ClusterDNSServer                 string
+	DinD                             bool
 	DockerDialerFactory              hosts.DialerFactory
 	EtcdHosts                        []*hosts.Host
 	EtcdReadyHosts                   []*hosts.Host
@@ -150,6 +151,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
 		RancherKubernetesEngineConfig: *rkeConfig,
 		ConfigPath:                    flags.ClusterFilePath,
 		ConfigDir:                     flags.ConfigDir,
+		DinD:                          flags.DinD,
 		StateFilePath:                 GetStateFilePath(flags.ClusterFilePath, flags.ConfigDir),
 		PrivateRegistriesMap:          make(map[string]v3.PrivateRegistry),
 	}

cluster/defaults.go

@@ -54,6 +54,7 @@ const (
 type ExternalFlags struct {
 	ConfigDir        string
 	ClusterFilePath  string
+	DinD             bool
 	DisablePortCheck bool
 	Local            bool
 	UpdateOnly       bool

cluster/plan.go

@@ -117,36 +117,48 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
 		c.getRKEToolsEntryPoint(),
 		"kube-apiserver",
 	}
-
+	baseEnabledAdmissionPlugins := []string{
+		"DefaultStorageClass",
+		"DefaultTolerationSeconds",
+		"LimitRanger",
+		"NamespaceLifecycle",
+		"NodeRestriction",
+		"PersistentVolumeLabel",
+		"ResourceQuota",
+		"ServiceAccount",
+	}
 	CommandArgs := map[string]string{
-		"insecure-bind-address":              "127.0.0.1",
-		"bind-address":                       "0.0.0.0",
-		"insecure-port":                      "0",
-		"secure-port":                        "6443",
-		"cloud-provider":                     c.CloudProvider.Name,
 		"allow-privileged":                   "true",
-		"kubelet-preferred-address-types":    "InternalIP,ExternalIP,Hostname",
+		"anonymous-auth":                     "false",
-		"service-cluster-ip-range":           c.Services.KubeAPI.ServiceClusterIPRange,
+		"bind-address":                       "0.0.0.0",
-		"service-node-port-range":            c.Services.KubeAPI.ServiceNodePortRange,
-		"storage-backend":                    "etcd3",
 		"client-ca-file":                     pki.GetCertPath(pki.CACertName),
-		"tls-cert-file":                      pki.GetCertPath(pki.KubeAPICertName),
+		"cloud-provider":                     c.CloudProvider.Name,
-		"tls-private-key-file":               pki.GetKeyPath(pki.KubeAPICertName),
-		"kubelet-client-certificate":         pki.GetCertPath(pki.KubeAPICertName),
-		"kubelet-client-key":                 pki.GetKeyPath(pki.KubeAPICertName),
-		"service-account-key-file":           pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
 		"etcd-cafile":                        etcdCAClientCert,
 		"etcd-certfile":                      etcdClientCert,
 		"etcd-keyfile":                       etcdClientKey,
-		"etcd-servers":                       etcdConnectionString,
 		"etcd-prefix":                        etcdPathPrefix,
-		"requestheader-client-ca-file":       pki.GetCertPath(pki.RequestHeaderCACertName),
+		"etcd-servers":                       etcdConnectionString,
-		"requestheader-allowed-names":        pki.APIProxyClientCertName,
+		"insecure-port":                      "0",
-		"proxy-client-key-file":              pki.GetKeyPath(pki.APIProxyClientCertName),
+		"kubelet-client-certificate":         pki.GetCertPath(pki.KubeAPICertName),
+		"kubelet-client-key":                 pki.GetKeyPath(pki.KubeAPICertName),
+		"kubelet-preferred-address-types":    "InternalIP,ExternalIP,Hostname",
+		"profiling":                          "false",
 		"proxy-client-cert-file":             pki.GetCertPath(pki.APIProxyClientCertName),
+		"proxy-client-key-file":              pki.GetKeyPath(pki.APIProxyClientCertName),
+		"requestheader-allowed-names":        pki.APIProxyClientCertName,
+		"requestheader-client-ca-file":       pki.GetCertPath(pki.RequestHeaderCACertName),
 		"requestheader-extra-headers-prefix": "X-Remote-Extra-",
 		"requestheader-group-headers":        "X-Remote-Group",
 		"requestheader-username-headers":     "X-Remote-User",
+		"repair-malformed-updates":           "false",
+		"secure-port":                        "6443",
+		"service-account-key-file":           pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
+		"service-account-lookup":             "true",
+		"service-cluster-ip-range":           c.Services.KubeAPI.ServiceClusterIPRange,
+		"service-node-port-range":            c.Services.KubeAPI.ServiceNodePortRange,
+		"storage-backend":                    "etcd3",
+		"tls-cert-file":                      pki.GetCertPath(pki.KubeAPICertName),
+		"tls-private-key-file":               pki.GetKeyPath(pki.KubeAPICertName),
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
 		CommandArgs["cloud-config"] = cloudConfigFileName
@@ -180,6 +192,28 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
 	if c.Authorization.Mode == services.RBACAuthorizationMode {
 		CommandArgs["authorization-mode"] = "Node,RBAC"
 	}
+
+	// PodSecurityPolicy
+	if c.Services.KubeAPI.PodSecurityPolicy {
+		CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
+		baseEnabledAdmissionPlugins = append(baseEnabledAdmissionPlugins, "PodSecurityPolicy")
+	}
+
+	// Admission control plugins
+	// Resolution order:
+	//   k8s_defaults.go K8sVersionServiceOptions
+	//   enabledAdmissionPlugins
+	//   cluster.yml extra_args overwrites it all
+	for _, optionName := range admissionControlOptionNames {
+		if _, ok := CommandArgs[optionName]; ok {
+			enabledAdmissionPlugins := strings.Split(CommandArgs[optionName], ",")
+			enabledAdmissionPlugins = append(enabledAdmissionPlugins, baseEnabledAdmissionPlugins...)
+
+			// Join unique slice as arg
+			CommandArgs[optionName] = strings.Join(util.UniqueStringSlice(enabledAdmissionPlugins), ",")
+			break
+		}
+	}
 	if c.Services.KubeAPI.PodSecurityPolicy {
 		CommandArgs["runtime-config"] = "extensions/v1beta1/podsecuritypolicy=true"
 		for _, optionName := range admissionControlOptionNames {
@@ -240,21 +274,27 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
 	}
 
 	CommandArgs := map[string]string{
-		"address":                          "0.0.0.0",
+		"address":                          "127.0.0.1",
-		"cloud-provider":                   c.CloudProvider.Name,
 		"allow-untagged-cloud":             "true",
+		"allocate-node-cidrs":              "true",
+		"cloud-provider":                   c.CloudProvider.Name,
+		"cluster-cidr":                     c.ClusterCIDR,
 		"configure-cloud-routes":           "false",
-		"leader-elect":                     "true",
-		"kubeconfig":                       pki.GetConfigPath(pki.KubeControllerCertName),
 		"enable-hostpath-provisioner":      "false",
+		"kubeconfig":                       pki.GetConfigPath(pki.KubeControllerCertName),
+		"leader-elect":                     "true",
 		"node-monitor-grace-period":        "40s",
 		"pod-eviction-timeout":             "5m0s",
-		"v":                                "2",
+		"profiling":                        "false",
-		"allocate-node-cidrs":              "true",
-		"cluster-cidr":                     c.ClusterCIDR,
-		"service-cluster-ip-range":         c.Services.KubeController.ServiceClusterIPRange,
-		"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
 		"root-ca-file":                     pki.GetCertPath(pki.CACertName),
+		"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
+		"service-cluster-ip-range":         c.Services.KubeController.ServiceClusterIPRange,
+		"terminated-pod-gc-threshold":      "1000",
+		"v":                                "2",
+	}
+	// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
+	if c.DinD {
+		CommandArgs["address"] = "0.0.0.0"
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
 		CommandArgs["cloud-config"] = cloudConfigFileName
@@ -332,29 +372,31 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
 	}
 
 	CommandArgs := map[string]string{
-		"v":                            "2",
 		"address":                           "0.0.0.0",
-		"cadvisor-port":                "0", //depricated in 1.12
-		"read-only-port":               "0",
-		"cluster-domain":               c.ClusterDomain,
-		"pod-infra-container-image":    c.Services.Kubelet.InfraContainerImage,
-		"cgroups-per-qos":              "True",
-		"enforce-node-allocatable":     "",
-		"hostname-override":            host.HostnameOverride,
-		"cluster-dns":                  c.ClusterDNSServer,
-		"network-plugin":               "cni",
-		"cni-conf-dir":                 "/etc/cni/net.d",
-		"cni-bin-dir":                  "/opt/cni/bin",
-		"resolv-conf":                  "/etc/resolv.conf",
 		"allow-privileged":                  "true",
-		"cloud-provider":               c.CloudProvider.Name,
-		"kubeconfig":                   pki.GetConfigPath(pki.KubeNodeCertName),
-		"client-ca-file":               pki.GetCertPath(pki.CACertName),
 		"anonymous-auth":                    "false",
-		"volume-plugin-dir":            "/var/lib/kubelet/volumeplugins",
-		"fail-swap-on":                 strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
-		"root-dir":                     path.Join(prefixPath, "/var/lib/kubelet"),
 		"authentication-token-webhook":      "true",
+		"cgroups-per-qos":                   "True",
+		"client-ca-file":                    pki.GetCertPath(pki.CACertName),
+		"cloud-provider":                    c.CloudProvider.Name,
+		"cluster-dns":                       c.ClusterDNSServer,
+		"cluster-domain":                    c.ClusterDomain,
+		"cni-bin-dir":                       "/opt/cni/bin",
+		"cni-conf-dir":                      "/etc/cni/net.d",
+		"enforce-node-allocatable":          "",
+		"event-qps":                         "0",
+		"fail-swap-on":                      strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
+		"hostname-override":                 host.HostnameOverride,
+		"kubeconfig":                        pki.GetConfigPath(pki.KubeNodeCertName),
+		"make-iptables-util-chains":         "true",
+		"network-plugin":                    "cni",
+		"pod-infra-container-image":         c.Services.Kubelet.InfraContainerImage,
+		"read-only-port":                    "0",
+		"resolv-conf":                       "/etc/resolv.conf",
+		"root-dir":                          path.Join(prefixPath, "/var/lib/kubelet"),
+		"streaming-connection-idle-timeout": "30m",
+		"volume-plugin-dir":                 "/var/lib/kubelet/volumeplugins",
+		"v":                                 "2",
 	}
 	if host.IsControl && !host.IsWorker {
 		CommandArgs["register-with-taints"] = unschedulableControlTaint
@@ -371,11 +413,11 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
 			fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile)))
 	}
 	if len(c.PrivateRegistriesMap) > 0 {
-		kubeletDcokerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap)
+		kubeletDockerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap)
 		c.Services.Kubelet.ExtraEnv = append(
 			c.Services.Kubelet.ExtraEnv,
 			fmt.Sprintf("%s=%s", KubeletDockerConfigEnv,
-				b64.StdEncoding.EncodeToString([]byte(kubeletDcokerConfig))))
+				b64.StdEncoding.EncodeToString([]byte(kubeletDockerConfig))))
 
 		c.Services.Kubelet.ExtraEnv = append(
 			c.Services.Kubelet.ExtraEnv,
@@ -468,11 +510,14 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string) v3.
 	CommandArgs := map[string]string{
 		"cluster-cidr":         c.ClusterCIDR,
 		"v":                    "2",
-		"healthz-bind-address": "0.0.0.0",
+		"healthz-bind-address": "127.0.0.1",
 		"hostname-override":    host.HostnameOverride,
 		"kubeconfig":           pki.GetConfigPath(pki.KubeProxyCertName),
 	}
-
+	// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
+	if c.DinD {
+		CommandArgs["healthz-bind-address"] = "0.0.0.0"
+	}
 	// check if our version has specific options for this component
 	serviceOptions := c.GetKubernetesServicesOptions()
 	if serviceOptions.Kubeproxy != nil {
@@ -566,10 +611,16 @@ func (c *Cluster) BuildSchedulerProcess(prefixPath string) v3.Process {
 	CommandArgs := map[string]string{
 		"leader-elect": "true",
 		"v":            "2",
-		"address":      "0.0.0.0",
+		"address":      "127.0.0.1",
+		"profiling":    "false",
 		"kubeconfig":   pki.GetConfigPath(pki.KubeSchedulerCertName),
 	}
 
+	// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
+	if c.DinD {
+		CommandArgs["address"] = "0.0.0.0"
+	}
+
 	// check if our version has specific options for this component
 	serviceOptions := c.GetKubernetesServicesOptions()
 	if serviceOptions.Scheduler != nil {

cmd/up.go

@@ -315,6 +315,7 @@ func clusterUpDind(ctx *cli.Context) error {
 	dialers := hosts.GetDialerOptions(hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil)
 	// setting up flags
 	flags := cluster.GetExternalFlags(false, false, disablePortCheck, "", filePath)
+	flags.DinD = true
 
 	if ctx.Bool("init") {
 		return ClusterInit(context.Background(), rkeConfig, dialers, flags)

util/util.go

@@ -37,3 +37,17 @@ func ErrList(e []error) error {
 	}
 	return nil
 }
+
+// UniqueStringSlice - Input slice, retrun slice with unique elements. Will not maintain order.
+func UniqueStringSlice(elements []string) []string {
+	encountered := map[string]bool{}
+	result := []string{}
+
+	for v := range elements {
+		if !encountered[elements[v]] {
+			encountered[elements[v]] = true
+			result = append(result, elements[v])
+		}
+	}
+	return result
+}