diff --git a/cluster/addons.go b/cluster/addons.go index c0078b28..2af3bc22 100644 --- a/cluster/addons.go +++ b/cluster/addons.go @@ -3,26 +3,27 @@ package cluster import ( "bytes" "context" + "encoding/json" "fmt" - rkeData "github.com/rancher/kontainer-driver-metadata/rke/templates" - "github.com/rancher/rke/templates" - "os" - "os/exec" - "time" - "io/ioutil" "net/http" + "os" + "os/exec" "strings" + "time" + rkeData "github.com/rancher/kontainer-driver-metadata/rke/templates" "github.com/rancher/rke/addons" "github.com/rancher/rke/authz" "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" "github.com/rancher/rke/services" + "github.com/rancher/rke/templates" "github.com/rancher/rke/util" "github.com/rancher/types/apis/management.cattle.io/v3" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" ) const ( @@ -58,6 +59,7 @@ type ingressOptions struct { AlpineImage string IngressImage string IngressBackend string + UpdateStrategy *appsv1.DaemonSetUpdateStrategy } type MetricsServerOptions struct { @@ -66,6 +68,8 @@ type MetricsServerOptions struct { NodeSelector map[string]string MetricsServerImage string Version string + UpdateStrategy *appsv1.DeploymentStrategy + Replicas *int32 } type CoreDNSOptions struct { @@ -77,6 +81,8 @@ type CoreDNSOptions struct { ReverseCIDRs []string UpstreamNameservers []string NodeSelector map[string]string + UpdateStrategy *appsv1.DeploymentStrategy + LinearAutoscalerParams string } type KubeDNSOptions struct { @@ -91,6 +97,8 @@ type KubeDNSOptions struct { UpstreamNameservers []string StubDomains map[string][]string NodeSelector map[string]string + UpdateStrategy *appsv1.DeploymentStrategy + LinearAutoscalerParams string } type addonError struct { @@ -270,7 +278,13 @@ func (c *Cluster) deployKubeDNS(ctx context.Context, data map[string]interface{} ReverseCIDRs: c.DNS.ReverseCIDRs, StubDomains: c.DNS.StubDomains, NodeSelector: c.DNS.NodeSelector, + UpdateStrategy: c.DNS.UpdateStrategy, } + linearModeBytes, err := json.Marshal(c.DNS.LinearAutoscalerParams) + if err != nil { + return err + } + KubeDNSConfig.LinearAutoscalerParams = string(linearModeBytes) tmplt, err := templates.GetVersionedTemplates(rkeData.KubeDNS, data, c.Version) if err != nil { return err @@ -297,7 +311,13 @@ func (c *Cluster) deployCoreDNS(ctx context.Context, data map[string]interface{} UpstreamNameservers: c.DNS.UpstreamNameservers, ReverseCIDRs: c.DNS.ReverseCIDRs, NodeSelector: c.DNS.NodeSelector, + UpdateStrategy: c.DNS.UpdateStrategy, } + linearModeBytes, err := json.Marshal(c.DNS.LinearAutoscalerParams) + if err != nil { + return err + } + CoreDNSConfig.LinearAutoscalerParams = string(linearModeBytes) tmplt, err := templates.GetVersionedTemplates(rkeData.CoreDNS, data, c.Version) if err != nil { return err @@ -341,6 +361,8 @@ func (c *Cluster) deployMetricServer(ctx context.Context, data map[string]interf Options: c.Monitoring.Options, NodeSelector: c.Monitoring.NodeSelector, Version: util.GetTagMajorVersion(versionTag), + UpdateStrategy: c.Monitoring.UpdateStrategy, + Replicas: c.Monitoring.Replicas, } tmplt, err := templates.GetVersionedTemplates(rkeData.MetricsServer, data, c.Version) if err != nil { @@ -498,6 +520,7 @@ func (c *Cluster) deployIngress(ctx context.Context, data map[string]interface{} ExtraEnvs: c.Ingress.ExtraEnvs, ExtraVolumes: c.Ingress.ExtraVolumes, ExtraVolumeMounts: c.Ingress.ExtraVolumeMounts, + UpdateStrategy: c.Ingress.UpdateStrategy, } // since nginx ingress controller 0.16.0, it can be run as non-root and doesn't require privileged anymore. // So we can use securityContext instead of setting privileges via initContainer. diff --git a/cluster/cluster.go b/cluster/cluster.go index 043f0cf4..32bd4451 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/api/types" ghodssyaml "github.com/ghodss/yaml" "github.com/rancher/norman/types/convert" + "github.com/rancher/norman/types/values" "github.com/rancher/rke/authz" "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" @@ -26,6 +27,7 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -97,6 +99,12 @@ const ( serviceAccountTokenFileParam = "service-account-key-file" SystemNamespace = "kube-system" + daemonsetType = "DaemonSet" + deploymentType = "Deployment" + ingressAddon = "ingress" + monitoringAddon = "monitoring" + dnsAddon = "dns" + networkAddon = "network" ) func (c *Cluster) DeployControlPlane(ctx context.Context, svcOptionData map[string]*v3.KubernetesServicesOptions, reconcileCluster bool) error { @@ -294,6 +302,51 @@ func parseAdmissionConfig(clusterFile string, rkeConfig *v3.RancherKubernetesEng return nil } +func parseAddonConfig(clusterFile string, rkeConfig *v3.RancherKubernetesEngineConfig) error { + var r map[string]interface{} + err := ghodssyaml.Unmarshal([]byte(clusterFile), &r) + if err != nil { + return fmt.Errorf("[parseAddonConfig] error unmarshalling RKE config: %v", err) + } + addonsResourceType := map[string]string{ + ingressAddon: daemonsetType, + networkAddon: daemonsetType, + monitoringAddon: deploymentType, + dnsAddon: deploymentType, + } + for addonName, addonType := range addonsResourceType { + updateStrategyField := values.GetValueN(r, addonName, "update_strategy") + if updateStrategyField == nil { + continue + } + switch addonType { + case daemonsetType: + updateStrategy, err := parseDaemonSetUpdateStrategy(updateStrategyField) + if err != nil { + return err + } + switch addonName { + case ingressAddon: + rkeConfig.Ingress.UpdateStrategy = updateStrategy + case networkAddon: + rkeConfig.Network.UpdateStrategy = updateStrategy + } + case deploymentType: + updateStrategy, err := parseDeploymentUpdateStrategy(updateStrategyField) + if err != nil { + return err + } + switch addonName { + case dnsAddon: + rkeConfig.DNS.UpdateStrategy = updateStrategy + case monitoringAddon: + rkeConfig.Monitoring.UpdateStrategy = updateStrategy + } + } + } + return nil +} + func parseIngressConfig(clusterFile string, rkeConfig *v3.RancherKubernetesEngineConfig) error { if &rkeConfig.Ingress == nil { return nil @@ -316,6 +369,32 @@ func parseIngressConfig(clusterFile string, rkeConfig *v3.RancherKubernetesEngin return nil } +func parseDaemonSetUpdateStrategy(updateStrategyField interface{}) (*appsv1.DaemonSetUpdateStrategy, error) { + updateStrategyBytes, err := json.Marshal(updateStrategyField) + if err != nil { + return nil, fmt.Errorf("[parseDaemonSetUpdateStrategy] error marshalling updateStrategy: %v", err) + } + var updateStrategy *appsv1.DaemonSetUpdateStrategy + err = json.Unmarshal(updateStrategyBytes, &updateStrategy) + if err != nil { + return nil, fmt.Errorf("[parseIngressUpdateStrategy] error unmarshaling updateStrategy: %v", err) + } + return updateStrategy, nil +} + +func parseDeploymentUpdateStrategy(updateStrategyField interface{}) (*appsv1.DeploymentStrategy, error) { + updateStrategyBytes, err := json.Marshal(updateStrategyField) + if err != nil { + return nil, fmt.Errorf("[parseDeploymentUpdateStrategy] error marshalling updateStrategy: %v", err) + } + var updateStrategy *appsv1.DeploymentStrategy + err = json.Unmarshal(updateStrategyBytes, &updateStrategy) + if err != nil { + return nil, fmt.Errorf("[parseDeploymentUpdateStrategy] error unmarshaling updateStrategy: %v", err) + } + return updateStrategy, nil +} + func parseIngressExtraEnv(ingressMap map[string]interface{}, rkeConfig *v3.RancherKubernetesEngineConfig) error { extraEnvs, ok := ingressMap["extra_envs"] if !ok { @@ -448,13 +527,15 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error) if err := parseAuditLogConfig(clusterFile, &rkeConfig); err != nil { return &rkeConfig, fmt.Errorf("error parsing audit log config: %v", err) } - if err := parseIngressConfig(clusterFile, &rkeConfig); err != nil { return &rkeConfig, fmt.Errorf("error parsing ingress config: %v", err) } if err := parseNodeDrainInput(clusterFile, &rkeConfig); err != nil { return &rkeConfig, fmt.Errorf("error parsing upgrade strategy and node drain input: %v", err) } + if err := parseAddonConfig(clusterFile, &rkeConfig); err != nil { + return &rkeConfig, fmt.Errorf("error parsing addon config: %v", err) + } return &rkeConfig, nil } diff --git a/cluster/defaults.go b/cluster/defaults.go index 1553fccd..d1fa5018 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -16,8 +16,10 @@ import ( "github.com/rancher/rke/util" v3 "github.com/rancher/types/apis/management.cattle.io/v3" "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" apiserverv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1" auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" ) @@ -83,6 +85,24 @@ const ( DefaultNodeDrainIgnoreDaemonsets = true ) +var ( + DefaultDaemonSetMaxUnavailable = intstr.FromInt(1) + DefaultDeploymentUpdateStrategyParams = intstr.FromString("25%") + DefaultDaemonSetUpdateStrategy = appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxUnavailable: &DefaultDaemonSetMaxUnavailable}, + } + DefaultDeploymentUpdateStrategy = appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &DefaultDeploymentUpdateStrategyParams, + MaxSurge: &DefaultDeploymentUpdateStrategyParams, + }, + } + DefaultClusterProportionalAutoscalerLinearParams = v3.LinearAutoscalerParams{CoresPerReplica: 128, NodesPerReplica: 4, Min: 1, PreventSinglePointFailure: true} + DefaultMonitoringAddonReplicas = int32(1) +) + type ExternalFlags struct { CertificateDir string ClusterFilePath string @@ -194,7 +214,7 @@ func (c *Cluster) setClusterDefaults(ctx context.Context, flags ExternalFlags) e c.setClusterNetworkDefaults() c.setClusterAuthnDefaults() c.setNodeUpgradeStrategy() - + c.setAddonsDefaults() return nil } @@ -568,3 +588,42 @@ func GetExternalFlags(local, updateOnly, disablePortCheck bool, configDir, clust ClusterFilePath: clusterFilePath, } } + +func (c *Cluster) setAddonsDefaults() { + c.Ingress.UpdateStrategy = setDaemonsetAddonDefaults(c.Ingress.UpdateStrategy) + c.Network.UpdateStrategy = setDaemonsetAddonDefaults(c.Network.UpdateStrategy) + c.DNS.UpdateStrategy = setDeploymentAddonDefaults(c.DNS.UpdateStrategy) + if c.DNS.LinearAutoscalerParams == nil { + c.DNS.LinearAutoscalerParams = &DefaultClusterProportionalAutoscalerLinearParams + } + c.Monitoring.UpdateStrategy = setDeploymentAddonDefaults(c.Monitoring.UpdateStrategy) + if c.Monitoring.Replicas == nil { + c.Monitoring.Replicas = &DefaultMonitoringAddonReplicas + } +} + +func setDaemonsetAddonDefaults(updateStrategy *appsv1.DaemonSetUpdateStrategy) *appsv1.DaemonSetUpdateStrategy { + if updateStrategy != nil && updateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { + return updateStrategy + } + if updateStrategy == nil || updateStrategy.RollingUpdate == nil || updateStrategy.RollingUpdate.MaxUnavailable == nil { + return &DefaultDaemonSetUpdateStrategy + } + return updateStrategy +} + +func setDeploymentAddonDefaults(updateStrategy *appsv1.DeploymentStrategy) *appsv1.DeploymentStrategy { + if updateStrategy != nil && updateStrategy.Type != appsv1.RollingUpdateDeploymentStrategyType { + return updateStrategy + } + if updateStrategy == nil || updateStrategy.RollingUpdate == nil { + return &DefaultDeploymentUpdateStrategy + } + if updateStrategy.RollingUpdate.MaxUnavailable == nil { + updateStrategy.RollingUpdate.MaxUnavailable = &DefaultDeploymentUpdateStrategyParams + } + if updateStrategy.RollingUpdate.MaxSurge == nil { + updateStrategy.RollingUpdate.MaxSurge = &DefaultDeploymentUpdateStrategyParams + } + return updateStrategy +} diff --git a/cluster/network.go b/cluster/network.go index a6c1754f..771e8de5 100644 --- a/cluster/network.go +++ b/cluster/network.go @@ -109,7 +109,8 @@ const ( RBACConfig = "RBACConfig" ClusterVersion = "ClusterVersion" - NodeSelector = "NodeSelector" + NodeSelector = "NodeSelector" + UpdateStrategy = "UpdateStrategy" ) var EtcdPortList = []string{ @@ -173,6 +174,7 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context, data map[string]interface RBACConfig: c.Authorization.Mode, ClusterVersion: util.GetTagMajorVersion(c.Version), NodeSelector: c.Network.NodeSelector, + UpdateStrategy: c.Network.UpdateStrategy, } pluginYaml, err := c.getNetworkPluginManifest(flannelConfig, data) if err != nil { @@ -196,6 +198,7 @@ func (c *Cluster) doCalicoDeploy(ctx context.Context, data map[string]interface{ RBACConfig: c.Authorization.Mode, NodeSelector: c.Network.NodeSelector, MTU: c.Network.MTU, + UpdateStrategy: c.Network.UpdateStrategy, } pluginYaml, err := c.getNetworkPluginManifest(calicoConfig, data) if err != nil { @@ -233,8 +236,9 @@ func (c *Cluster) doCanalDeploy(ctx context.Context, data map[string]interface{} "VNI": flannelVni, "Port": flannelPort, }, - NodeSelector: c.Network.NodeSelector, - MTU: c.Network.MTU, + NodeSelector: c.Network.NodeSelector, + MTU: c.Network.MTU, + UpdateStrategy: c.Network.UpdateStrategy, } pluginYaml, err := c.getNetworkPluginManifest(canalConfig, data) if err != nil { @@ -253,6 +257,7 @@ func (c *Cluster) doWeaveDeploy(ctx context.Context, data map[string]interface{} RBACConfig: c.Authorization.Mode, NodeSelector: c.Network.NodeSelector, MTU: c.Network.MTU, + UpdateStrategy: c.Network.UpdateStrategy, } pluginYaml, err := c.getNetworkPluginManifest(weaveConfig, data) if err != nil {