Add metrics-server addon deployment

addons/kubedns.go

@@ -9,7 +9,9 @@ const (
 	KubeDNSAutoScalerImage = "KubeDNSAutoScalerImage"
 	KubeDNSServer          = "ClusterDNSServer"
 	KubeDNSClusterDomain   = "ClusterDomain"
+	MetricsServerImage     = "MetricsServerImage"
 	RBAC                   = "RBAC"
+	MetricsServerOptions   = "MetricsServerOptions"
 )
 
 func GetKubeDNSManifest(kubeDNSConfig map[string]string) (string, error) {

addons/metrics.go

@@ -0,0 +1,8 @@
+package addons
+
+import "github.com/rancher/rke/templates"
+
+func GetMetricsServerManifest(MetricsServerConfig interface{}) (string, error) {
+
+	return templates.CompileTemplateFromMap(templates.MetricsServerTemplate, MetricsServerConfig)
+}

cluster/addons.go

@@ -25,8 +25,9 @@ const (
 	IngressAddonResourceName      = "rke-ingress-controller"
 	UserAddonsIncludeResourceName = "rke-user-includes-addons"
 
-	IngressAddonJobName       = "rke-ingress-controller-deploy-job"
-	IngressAddonDeleteJobName = "rke-ingress-controller-delete-job"
+	IngressAddonJobName            = "rke-ingress-controller-deploy-job"
+	IngressAddonDeleteJobName      = "rke-ingress-controller-delete-job"
+	MetricsServerAddonResourceName = "rke-metrics-addon"
 )
 
 type ingressOptions struct {
@@ -39,6 +40,12 @@ type ingressOptions struct {
 	IngressBackend string
 }
 
+type MetricsServerOptions struct {
+	RBACConfig         string
+	Options            map[string]string
+	MetricsServerImage string
+}
+
 type addonError struct {
 	err        string
 	isCritical bool
@@ -55,6 +62,14 @@ func (c *Cluster) deployK8sAddOns(ctx context.Context) error {
 		}
 		log.Warnf(ctx, "Failed to deploy addon execute job [%s]: %v", KubeDNSAddonResourceName, err)
 	}
+	if c.Monitoring.Provider == DefaultMonitoringProvider {
+		if err := c.deployMetricServer(ctx); err != nil {
+			if err, ok := err.(*addonError); ok && err.isCritical {
+				return err
+			}
+			log.Warnf(ctx, "Failed to deploy addon execute job [%s]: %v", MetricsServerAddonResourceName, err)
+		}
+	}
 	if err := c.deployIngress(ctx); err != nil {
 		if err, ok := err.(*addonError); ok && err.isCritical {
 			return err
@@ -187,6 +202,24 @@ func (c *Cluster) deployKubeDNS(ctx context.Context) error {
 	return nil
 }
 
+func (c *Cluster) deployMetricServer(ctx context.Context) error {
+	log.Infof(ctx, "[addons] Setting up Metrics Server")
+	MetricsServerConfig := MetricsServerOptions{
+		MetricsServerImage: c.SystemImages.MetricsServer,
+		RBACConfig:         c.Authorization.Mode,
+		Options:            c.Monitoring.Options,
+	}
+	kubeDNSYaml, err := addons.GetMetricsServerManifest(MetricsServerConfig)
+	if err != nil {
+		return err
+	}
+	if err := c.doAddonDeploy(ctx, kubeDNSYaml, MetricsServerAddonResourceName, false); err != nil {
+		return err
+	}
+	log.Infof(ctx, "[addons] KubeDNS deployed successfully..")
+	return nil
+}
+
 func (c *Cluster) deployWithKubectl(ctx context.Context, addonYaml string) error {
 	buf := bytes.NewBufferString(addonYaml)
 	cmd := exec.Command("kubectl", "--kubeconfig", c.LocalKubeConfigPath, "apply", "-f", "-")

cluster/certificates.go

@@ -23,6 +23,14 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 		var err error
 		if currentCluster != nil {
 			kubeCluster.Certificates = currentCluster.Certificates
+			// this is the case of handling upgrades for API server aggregation layer ca cert and API server proxy client key and cert
+			if kubeCluster.Certificates[pki.RequestHeaderCACertName].Certificate == nil {
+
+				kubeCluster.Certificates, err = regenerateAPIAggregationCerts(kubeCluster, kubeCluster.Certificates)
+				if err != nil {
+					return fmt.Errorf("Failed to regenerate Aggregation layer certificates %v", err)
+				}
+			}
 		} else {
 			var backupPlane string
 			var backupHosts []*hosts.Host
@@ -67,6 +75,14 @@ func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Clust
 						return fmt.Errorf("Failed to regenerate KubeAPI certificate %v", err)
 					}
 				}
+				// this is the case of handling upgrades for API server aggregation layer ca cert and API server proxy client key and cert
+				if kubeCluster.Certificates[pki.RequestHeaderCACertName].Certificate == nil {
+
+					kubeCluster.Certificates, err = regenerateAPIAggregationCerts(kubeCluster, kubeCluster.Certificates)
+					if err != nil {
+						return fmt.Errorf("Failed to regenerate Aggregation layer certificates %v", err)
+					}
+				}
 				return nil
 			}
 			log.Infof(ctx, "[certificates] No Certificate backup found on [%s] hosts", backupPlane)
@@ -293,3 +309,21 @@ func (c *Cluster) getBackupHosts() []*hosts.Host {
 	}
 	return backupHosts
 }
+
+func regenerateAPIAggregationCerts(c *Cluster, certificates map[string]pki.CertificatePKI) (map[string]pki.CertificatePKI, error) {
+	logrus.Debugf("[certificates] Regenerating Kubernetes API server aggregation layer requestheader client CA certificates")
+	requestHeaderCACrt, requestHeaderCAKey, err := pki.GenerateCACertAndKey(pki.RequestHeaderCACertName)
+	if err != nil {
+		return nil, err
+	}
+	certificates[pki.RequestHeaderCACertName] = pki.ToCertObject(pki.RequestHeaderCACertName, "", "", requestHeaderCACrt, requestHeaderCAKey)
+
+	//generate API server proxy client key and certs
+	logrus.Debugf("[certificates] Regenerating Kubernetes API server porxy client certificates")
+	apiserverProxyClientCrt, apiserverProxyClientKey, err := pki.GenerateSignedCertAndKey(requestHeaderCACrt, requestHeaderCAKey, true, pki.APIProxyClientCertName, nil, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	certificates[pki.APIProxyClientCertName] = pki.ToCertObject(pki.APIProxyClientCertName, "", "", apiserverProxyClientCrt, apiserverProxyClientKey)
+	return certificates, nil
+}

cluster/defaults.go

@@ -32,6 +32,7 @@ const (
 	DefaultIngressController         = "nginx"
 	DefaultEtcdBackupCreationPeriod  = "5m0s"
 	DefaultEtcdBackupRetentionPeriod = "24h"
+	DefaultMonitoringProvider        = "metrics-server"
 )
 
 func setDefaultIfEmptyMapValue(configMap map[string]string, key string, value string) {
@@ -103,6 +104,9 @@ func (c *Cluster) setClusterDefaults(ctx context.Context) {
 	if c.AddonJobTimeout == 0 {
 		c.AddonJobTimeout = k8s.DefaultTimeout
 	}
+	if len(c.Monitoring.Provider) == 0 {
+		c.Monitoring.Provider = DefaultMonitoringProvider
+	}
 	c.setClusterImageDefaults()
 	c.setClusterServicesDefaults()
 	c.setClusterNetworkDefaults()
@@ -164,6 +168,7 @@ func (c *Cluster) setClusterImageDefaults() {
 		&c.SystemImages.WeaveCNI:                  imageDefaults.WeaveCNI,
 		&c.SystemImages.Ingress:                   imageDefaults.Ingress,
 		&c.SystemImages.IngressBackend:            imageDefaults.IngressBackend,
+		&c.SystemImages.MetricsServer:             imageDefaults.MetricsServer,
 	}
 
 	for k, v := range systemImagesDefaultsMap {

cluster/plan.go

@@ -107,32 +107,35 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
 	}
 
 	CommandArgs := map[string]string{
-		"insecure-bind-address":           "127.0.0.1",
-		"bind-address":                    "0.0.0.0",
-		"insecure-port":                   "0",
-		"secure-port":                     "6443",
-		"cloud-provider":                  c.CloudProvider.Name,
-		"allow-privileged":                "true",
-		"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
-		"service-cluster-ip-range":        c.Services.KubeAPI.ServiceClusterIPRange,
-		"service-node-port-range":         c.Services.KubeAPI.ServiceNodePortRange,
-		"admission-control":               "ServiceAccount,NamespaceLifecycle,LimitRanger,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
-		"storage-backend":                 "etcd3",
-		"client-ca-file":                  pki.GetCertPath(pki.CACertName),
-		"tls-cert-file":                   pki.GetCertPath(pki.KubeAPICertName),
-		"tls-private-key-file":            pki.GetKeyPath(pki.KubeAPICertName),
-		"kubelet-client-certificate":      pki.GetCertPath(pki.KubeAPICertName),
-		"kubelet-client-key":              pki.GetKeyPath(pki.KubeAPICertName),
-		"service-account-key-file":        pki.GetKeyPath(pki.KubeAPICertName),
-		"etcd-cafile":                     etcdCAClientCert,
-		"etcd-certfile":                   etcdClientCert,
-		"etcd-keyfile":                    etcdClientKey,
-		"etcd-servers":                    etcdConnectionString,
-		"etcd-prefix":                     etcdPathPrefix,
-		"requestheader-client-ca-file":    pki.GetCertPath(pki.RequestHeaderCACertName),
-		"requestheader-allowed-names":     pki.APIProxyClientCertName,
-		"proxy-client-key-file":           pki.GetKeyPath(pki.APIProxyClientCertName),
-		"proxy-client-cert-file":          pki.GetCertPath(pki.APIProxyClientCertName),
+		"insecure-bind-address":              "127.0.0.1",
+		"bind-address":                       "0.0.0.0",
+		"insecure-port":                      "0",
+		"secure-port":                        "6443",
+		"cloud-provider":                     c.CloudProvider.Name,
+		"allow-privileged":                   "true",
+		"kubelet-preferred-address-types":    "InternalIP,ExternalIP,Hostname",
+		"service-cluster-ip-range":           c.Services.KubeAPI.ServiceClusterIPRange,
+		"service-node-port-range":            c.Services.KubeAPI.ServiceNodePortRange,
+		"admission-control":                  "ServiceAccount,NamespaceLifecycle,LimitRanger,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+		"storage-backend":                    "etcd3",
+		"client-ca-file":                     pki.GetCertPath(pki.CACertName),
+		"tls-cert-file":                      pki.GetCertPath(pki.KubeAPICertName),
+		"tls-private-key-file":               pki.GetKeyPath(pki.KubeAPICertName),
+		"kubelet-client-certificate":         pki.GetCertPath(pki.KubeAPICertName),
+		"kubelet-client-key":                 pki.GetKeyPath(pki.KubeAPICertName),
+		"service-account-key-file":           pki.GetKeyPath(pki.KubeAPICertName),
+		"etcd-cafile":                        etcdCAClientCert,
+		"etcd-certfile":                      etcdClientCert,
+		"etcd-keyfile":                       etcdClientKey,
+		"etcd-servers":                       etcdConnectionString,
+		"etcd-prefix":                        etcdPathPrefix,
+		"requestheader-client-ca-file":       pki.GetCertPath(pki.RequestHeaderCACertName),
+		"requestheader-allowed-names":        pki.APIProxyClientCertName,
+		"proxy-client-key-file":              pki.GetKeyPath(pki.APIProxyClientCertName),
+		"proxy-client-cert-file":             pki.GetCertPath(pki.APIProxyClientCertName),
+		"requestheader-extra-headers-prefix": "X-Remote-Extra-",
+		"requestheader-group-headers":        "X-Remote-Group",
+		"requestheader-username-headers":     "X-Remote-User",
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
 		CommandArgs["cloud-config"] = CloudConfigPath
@@ -299,28 +302,29 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
 	}
 
 	CommandArgs := map[string]string{
-		"v":                         "2",
-		"address":                   "0.0.0.0",
-		"cadvisor-port":             "0",
-		"read-only-port":            "0",
-		"cluster-domain":            c.ClusterDomain,
-		"pod-infra-container-image": c.Services.Kubelet.InfraContainerImage,
-		"cgroups-per-qos":           "True",
-		"enforce-node-allocatable":  "",
-		"hostname-override":         host.HostnameOverride,
-		"cluster-dns":               c.ClusterDNSServer,
-		"network-plugin":            "cni",
-		"cni-conf-dir":              "/etc/cni/net.d",
-		"cni-bin-dir":               "/opt/cni/bin",
-		"resolv-conf":               "/etc/resolv.conf",
-		"allow-privileged":          "true",
-		"cloud-provider":            c.CloudProvider.Name,
-		"kubeconfig":                pki.GetConfigPath(pki.KubeNodeCertName),
-		"client-ca-file":            pki.GetCertPath(pki.CACertName),
-		"anonymous-auth":            "false",
-		"volume-plugin-dir":         path.Join(prefixPath, "/var/lib/kubelet/volumeplugins"),
-		"fail-swap-on":              strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
-		"root-dir":                  path.Join(prefixPath, "/var/lib/kubelet"),
+		"v":                            "2",
+		"address":                      "0.0.0.0",
+		"cadvisor-port":                "0",
+		"read-only-port":               "0",
+		"cluster-domain":               c.ClusterDomain,
+		"pod-infra-container-image":    c.Services.Kubelet.InfraContainerImage,
+		"cgroups-per-qos":              "True",
+		"enforce-node-allocatable":     "",
+		"hostname-override":            host.HostnameOverride,
+		"cluster-dns":                  c.ClusterDNSServer,
+		"network-plugin":               "cni",
+		"cni-conf-dir":                 "/etc/cni/net.d",
+		"cni-bin-dir":                  "/opt/cni/bin",
+		"resolv-conf":                  "/etc/resolv.conf",
+		"allow-privileged":             "true",
+		"cloud-provider":               c.CloudProvider.Name,
+		"kubeconfig":                   pki.GetConfigPath(pki.KubeNodeCertName),
+		"client-ca-file":               pki.GetCertPath(pki.CACertName),
+		"anonymous-auth":               "false",
+		"volume-plugin-dir":            path.Join(prefixPath, "/var/lib/kubelet/volumeplugins"),
+		"fail-swap-on":                 strconv.FormatBool(c.Services.Kubelet.FailSwapOn),
+		"root-dir":                     path.Join(prefixPath, "/var/lib/kubelet"),
+		"authentication-token-webhook": "true",
 	}
 	if host.IsControl && !host.IsWorker {
 		CommandArgs["register-with-taints"] = unschedulableControlTaint

pki/deploy.go

@@ -137,7 +137,9 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 		certificate := CertificatePKI{}
 		crt, err := FetchFileFromHost(ctx, GetCertTempPath(certName), image, host, prsMap)
 		// I will only exit with an error if it's not a not-found-error and this is not an etcd certificate
-		if err != nil && !strings.HasPrefix(certName, "kube-etcd") {
+		if err != nil && (!strings.HasPrefix(certName, "kube-etcd") &&
+			!strings.Contains(certName, APIProxyClientCertName) &&
+			!strings.Contains(certName, RequestHeaderCACertName)) {
 			if strings.Contains(err.Error(), "no such file or directory") ||
 				strings.Contains(err.Error(), "Could not find the file") {
 				return nil, nil
@@ -145,8 +147,10 @@ func FetchCertificatesFromHost(ctx context.Context, extraHosts []*hosts.Host, ho
 			return nil, err
 
 		}
-		// If I can't find an etcd cert, I will not fail and will create it later.
-		if crt == "" && strings.HasPrefix(certName, "kube-etcd") {
+		// If I can't find an etcd or api aggregator cert, I will not fail and will create it later.
+		if crt == "" && (strings.HasPrefix(certName, "kube-etcd") ||
+			strings.Contains(certName, APIProxyClientCertName) ||
+			strings.Contains(certName, RequestHeaderCACertName)) {
 			tmpCerts[certName] = CertificatePKI{}
 			continue
 		}

pki/pki.go

@@ -43,7 +43,7 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 	certs := make(map[string]CertificatePKI)
 	// generate CA certificate and key
 	log.Infof(ctx, "[certificates] Generating CA kubernetes certificates")
-	caCrt, caKey, err := generateCACertAndKey(CACertName)
+	caCrt, caKey, err := GenerateCACertAndKey(CACertName)
 	if err != nil {
 		return nil, err
 	}
@@ -54,6 +54,7 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 	kubernetesServiceIP, err := GetKubernetesServiceIP(rkeConfig.Services.KubeAPI.ServiceClusterIPRange)
 	clusterDomain := rkeConfig.Services.Kubelet.ClusterDomain
 	cpHosts := hosts.NodesToHosts(rkeConfig.Nodes, controlRole)
+	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to get Kubernetes Service IP: %v", err)
 	}
@@ -88,7 +89,6 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 	}
 	certs[KubeProxyCertName] = ToCertObject(KubeProxyCertName, "", "", kubeProxyCrt, kubeProxyKey)
 
-	// generate Kubelet certificate and key
 	log.Infof(ctx, "[certificates] Generating Node certificate")
 	nodeCrt, nodeKey, err := GenerateSignedCertAndKey(caCrt, caKey, false, KubeNodeCommonName, nil, nil, []string{KubeNodeOrganizationName})
 	if err != nil {
@@ -139,7 +139,6 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 		}
 		certs[EtcdClientCACertName] = ToCertObject(EtcdClientCACertName, "", "", caCert[0], nil)
 	}
-	etcdHosts := hosts.NodesToHosts(rkeConfig.Nodes, etcdRole)
 	etcdAltNames := GetAltNames(etcdHosts, clusterDomain, kubernetesServiceIP, []string{})
 	for _, host := range etcdHosts {
 		log.Infof(ctx, "[certificates] Generating etcd-%s certificate and key", host.InternalAddress)
@@ -153,7 +152,7 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 
 	// generate request header client CA certificate and key
 	log.Infof(ctx, "[certificates] Generating Kubernetes API server aggregation layer requestheader client CA certificates")
-	requestHeaderCACrt, requestHeaderCAKey, err := generateCACertAndKey(RequestHeaderCACertName)
+	requestHeaderCACrt, requestHeaderCAKey, err := GenerateCACertAndKey(RequestHeaderCACertName)
 	if err != nil {
 		return nil, err
 	}
@@ -161,7 +160,7 @@ func GenerateRKECerts(ctx context.Context, rkeConfig v3.RancherKubernetesEngineC
 
 	//generate API server proxy client key and certs
 	log.Infof(ctx, "[certificates] Generating Kubernetes API server porxy client certificates")
-	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(requestHeaderCACrt, requestHeaderCAKey, false, APIProxyClientCertName, nil, nil, nil)
+	apiserverProxyClientCrt, apiserverProxyClientKey, err := GenerateSignedCertAndKey(requestHeaderCACrt, requestHeaderCAKey, true, APIProxyClientCertName, nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}

pki/util.go

@@ -51,7 +51,7 @@ func GenerateSignedCertAndKey(
 	return clientCert, rootKey, nil
 }
 
-func generateCACertAndKey(commonName string) (*x509.Certificate, *rsa.PrivateKey, error) {
+func GenerateCACertAndKey(commonName string) (*x509.Certificate, *rsa.PrivateKey, error) {
 	rootKey, err := cert.NewPrivateKey()
 	if err != nil {
 		return nil, nil, fmt.Errorf("Failed to generate private key for CA certificate: %v", err)

templates/metrics.go

@@ -0,0 +1,135 @@
+package templates
+
+const MetricsServerTemplate = `
+{{- if eq .RBACConfig "rbac"}}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "extensions"
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+{{- end }}
+---
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      containers:
+      - name: metrics-server
+        image: {{ .MetricsServerImage }}
+        imagePullPolicy: Always
+        command:
+        - /metrics-server
+        - --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true
+        {{ range $k,$v := .Options }}
+        -  --{{ $k }}={{ $v }}
+        {{ end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: 443
+`