From 14c239c598ee95b7adfc9071d1563a34b002109e Mon Sep 17 00:00:00 2001 From: rawmind0 Date: Fri, 13 Apr 2018 18:06:09 +0200 Subject: [PATCH] Added extra_args type map[string]string to ingress-controller. Added rancher-minimal-ssl.yml and rancher-minimal-passthrough.yml to deploy rancher v2.0 using rke. Updated README.md --- README.md | 39 +++++- cluster.yml | 2 + cluster/addons.go | 2 + ...mal.yml => rancher-minimal-passthrough.yml | 27 ++-- rancher-minimal-ssl.yml | 121 ++++++++++++++++++ templates/nginx-ingress.go | 3 + vendor.conf | 4 +- .../management.cattle.io/v3/cluster_types.go | 26 ++-- .../management.cattle.io/v3/k8s_defaults.go | 11 +- .../management.cattle.io/v3/logging_types.go | 11 +- .../management.cattle.io/v3/machine_types.go | 22 ++++ .../apis/management.cattle.io/v3/rke_types.go | 2 + .../v3/zz_generated_deepcopy.go | 70 ++++++++++ 13 files changed, 305 insertions(+), 35 deletions(-) rename rancher-minimal.yml => rancher-minimal-passthrough.yml (68%) create mode 100644 rancher-minimal-ssl.yml diff --git a/README.md b/README.md index effae34e..bb83236e 100644 --- a/README.md +++ b/README.md @@ -201,13 +201,15 @@ RKE will ask some questions around the cluster file like number of the hosts, ip ## Ingress Controller -RKE will deploy Nginx controller by default, user can disable this by specifying `none` to ingress `provider` option in the cluster configuration, user also can specify list of options for nginx config map listed in this [doc](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/configmap.md), for example: +RKE will deploy Nginx controller by default, user can disable this by specifying `none` to ingress `provider` option in the cluster configuration, user also can specify list of options for nginx config map listed in this [doc](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/configmap.md), and command line extra_args listed in this [doc](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/cli-arguments.md), for example: ``` ingress: provider: nginx options: map-hash-bucket-size: "128" ssl-protocols: SSLv2 + extra_args: + enable-ssl-passthrough: "" ``` By default, RKE will deploy ingress controller on all schedulable nodes (controlplane and workers), to specify only certain nodes for ingress controller to be deployed, user has to specify `node_selector` for the ingress and the right label on the node, for example: ``` @@ -335,11 +337,40 @@ nodes: ``` ## Deploying Rancher 2.0 using rke -Using RKE's pluggable user addons, it's possible to deploy Rancher 2.0 server with a single command after updating the node settings in the [rancher-minimal.yml](https://github.com/rancher/rke/blob/master/rancher-minimal.yml) cluster configuration: +Using RKE's pluggable user addons, it's possible to deploy Rancher 2.0 server in HA with a single command. + +Depending how you want to manage your ssl certificates, there are 2 deployment options: + +- Use own ssl cerficiates: + - Use [rancher-minimal-ssl.yml](https://github.com/rancher/rke/blob/master/rancher-minimal-ssl.yml) + - Update `nodes` configuration. + - Update at `cattle-ingress-http` ingress definition. FQDN should be a dns a entry pointing to all nodes IP's running ingress-controller (controlplane and workers by default). + - Update certificate, key and ca crt at `cattle-keys-server` secret, , and . Content must be in base64 format, `cat | base64` + - Update ssl certificate and key at `cattle-keys-ingress` secret, and . Content must be in base64 format, `cat | base64`. If selfsigned, certificate and key must be signed by same CA. + - Run RKE. + + ```bash + rke up --config rancher-minimal-ssl.yml + ``` + +- Use SSL-passthrough: + - Use [rancher-minimal-passthrough.yml](https://github.com/rancher/rke/blob/master/rancher-minimal-passthrough.yml) + - Update `nodes` configuration. + - Update FQDN at `cattle-ingress-http` ingress definition. FQDN should be a dns a entry, pointing to all nodes IP's running ingress-controller (controlplane and workers by default). + - Run RKE. + + ```bash + rke up --config rancher-minimal-passthrough.yml + ``` + +Once RKE execution finish, rancher is deployed at `cattle-system` namespace. You could access to your rancher instance by `https://` + +By default, rancher deployment has just 1 replica, scale it to desired replicas. -```bash -rke up --config rancher-minimal.yml ``` +kubectl -n cattle-system scale deployment cattle --replicas=3 +``` + ## Operating Systems Notes ### Atomic OS diff --git a/cluster.yml b/cluster.yml index 8fbade14..93d9fb7c 100644 --- a/cluster.yml +++ b/cluster.yml @@ -174,6 +174,8 @@ private_registries: # provider: nginx # node_selector: # app: ingress +# extra_args: +# enable-ssl-passthrough: "" ingress: provider: nginx diff --git a/cluster/addons.go b/cluster/addons.go index 65855c9b..c434fd5d 100644 --- a/cluster/addons.go +++ b/cluster/addons.go @@ -30,6 +30,7 @@ type ingressOptions struct { RBACConfig string Options map[string]string NodeSelector map[string]string + ExtraArgs map[string]string AlpineImage string IngressImage string IngressBackend string @@ -249,6 +250,7 @@ func (c *Cluster) deployIngress(ctx context.Context) error { RBACConfig: c.Authorization.Mode, Options: c.Ingress.Options, NodeSelector: c.Ingress.NodeSelector, + ExtraArgs: c.Ingress.ExtraArgs, AlpineImage: c.SystemImages.Alpine, IngressImage: c.SystemImages.Ingress, IngressBackend: c.SystemImages.IngressBackend, diff --git a/rancher-minimal.yml b/rancher-minimal-passthrough.yml similarity index 68% rename from rancher-minimal.yml rename to rancher-minimal-passthrough.yml index 7b147495..568685d7 100644 --- a/rancher-minimal.yml +++ b/rancher-minimal-passthrough.yml @@ -1,9 +1,15 @@ # default k8s version: v1.8.9-rancher1-1 # default network plugin: flannel nodes: - - address: 1.2.3.4 - user: ubuntu + - address: + user: role: [controlplane,etcd,worker] + ssh_key_path: + +ingress: + provider: nginx + extra_args: + enable-ssl-passthrough: "" addons: |- --- @@ -50,16 +56,19 @@ addons: |- metadata: namespace: cattle-system name: cattle-ingress-http + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: "30" + nginx.ingress.kubernetes.io/proxy-read-timeout: "1800" # Max time in seconds for ws to remain shell window open + nginx.ingress.kubernetes.io/proxy-send-timeout: "1800" # Max time in seconds for ws to remain shell window open + nginx.ingress.kubernetes.io/ssl-passthrough: "true" # Enable ssl-passthrough to backend. spec: rules: - - http: + - host: # FQDN to access cattle server + http: paths: - - backend: - serviceName: cattle-service - servicePort: 80 - - backend: - serviceName: cattle-service - servicePort: 443 + - backend: + serviceName: cattle-service + servicePort: 443 --- kind: Deployment apiVersion: extensions/v1beta1 diff --git a/rancher-minimal-ssl.yml b/rancher-minimal-ssl.yml new file mode 100644 index 00000000..ced7c4b2 --- /dev/null +++ b/rancher-minimal-ssl.yml @@ -0,0 +1,121 @@ +# default k8s version: v1.8.9-rancher1-1 +# default network plugin: flannel +nodes: + - address: + user: + role: [controlplane,etcd,worker] + ssh_key_path: + +addons: |- + --- + kind: Namespace + apiVersion: v1 + metadata: + name: cattle-system + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: cattle-crb + subjects: + - kind: User + name: system:serviceaccount:cattle-system:default + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: Secret + metadata: + name: cattle-keys-ingress + namespace: cattle-system + type: Opaque + data: + tls.crt: # ssl cert for ingress. If selfsigned, must be signed by same CA as cattle server + tls.key: # ssl key for ingress. If selfsigned, must be signed by same CA as cattle server + --- + apiVersion: v1 + kind: Secret + metadata: + name: cattle-keys-server + namespace: cattle-system + type: Opaque + data: + cert.pem: # ssl cert for cattle server. + key.pem: # ssl key for cattle server. + cacerts.pem: # CA cert used to sign cattle server cert and key + --- + apiVersion: v1 + kind: Service + metadata: + namespace: cattle-system + name: cattle-service + labels: + app: cattle + spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + - port: 443 + targetPort: 443 + protocol: TCP + name: https + selector: + app: cattle + --- + apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + namespace: cattle-system + name: cattle-ingress-http + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: "30" + nginx.ingress.kubernetes.io/proxy-read-timeout: "1800" # Max time in seconds for ws to remain shell window open + nginx.ingress.kubernetes.io/proxy-send-timeout: "1800" # Max time in seconds for ws to remain shell window open + spec: + rules: + - host: # FQDN to access cattle server + http: + paths: + - backend: + serviceName: cattle-service + servicePort: 80 + tls: + - secretName: cattle-keys-ingress + hosts: + - # FQDN to access cattle server + --- + kind: Deployment + apiVersion: extensions/v1beta1 + metadata: + namespace: cattle-system + name: cattle + spec: + replicas: 1 + template: + metadata: + labels: + app: cattle + spec: + containers: + - image: rancher/server:master + imagePullPolicy: Always + name: cattle-server + ports: + - containerPort: 80 + protocol: TCP + - containerPort: 443 + protocol: TCP + volumeMounts: + - mountPath: /etc/rancher/ssl + name: cattle-keys-volume + readOnly: true + volumes: + - name: cattle-keys-volume + secret: + defaultMode: 420 + secretName: cattle-keys-server diff --git a/templates/nginx-ingress.go b/templates/nginx-ingress.go index c44f4f18..ffba6ad3 100644 --- a/templates/nginx-ingress.go +++ b/templates/nginx-ingress.go @@ -203,6 +203,9 @@ spec: - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --annotations-prefix=nginx.ingress.kubernetes.io + {{ range $k, $v := .ExtraArgs }} + - --{{ $k }}{{if ne $v "" }}={{ $v }}{{end}} + {{ end }} env: - name: POD_NAME valueFrom: diff --git a/vendor.conf b/vendor.conf index c6dc2ef8..a1c57683 100644 --- a/vendor.conf +++ b/vendor.conf @@ -18,10 +18,10 @@ gopkg.in/check.v1 11d3bc7aa68e238947792f30573146a3231fc0f k8s.io/api/core/v1 4df58c811fe2e65feb879227b2b245e4dc26e7ad k8s.io/client-go v5.0.0 transitive=true github.com/gorilla/websocket v1.2.0 -golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 +golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 github.com/coreos/etcd 52f73c5a6cb0d1d196ffd6eced406c9d8502078a transitive=true github.com/coreos/go-semver e214231b295a8ea9479f11b70b35d5acf3556d9b github.com/ugorji/go/codec ccfe18359b55b97855cee1d3f74e5efbda4869dc github.com/rancher/norman ff60298f31f081b06d198815b4c178a578664f7d -github.com/rancher/types 574e26d2fb850f15b9269a54cacb9467505d44c5 +github.com/rancher/types 32ed9ccfe5b3ffd6acff15e354c0dd713f5b88d7 diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/cluster_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/cluster_types.go index d16a6ce1..89ece733 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/cluster_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/cluster_types.go @@ -52,16 +52,17 @@ type Cluster struct { } type ClusterSpec struct { - DisplayName string `json:"displayName"` - Description string `json:"description"` - Internal bool `json:"internal" norman:"nocreate,noupdate"` - DesiredAgentImage string `json:"desiredAgentImage"` - ImportedConfig *ImportedConfig `json:"importedConfig,omitempty" norman:"nocreate,noupdate"` - GoogleKubernetesEngineConfig *GoogleKubernetesEngineConfig `json:"googleKubernetesEngineConfig,omitempty"` - AzureKubernetesServiceConfig *AzureKubernetesServiceConfig `json:"azureKubernetesServiceConfig,omitempty"` - RancherKubernetesEngineConfig *RancherKubernetesEngineConfig `json:"rancherKubernetesEngineConfig,omitempty"` - DefaultPodSecurityPolicyTemplateName string `json:"defaultPodSecurityPolicyTemplateName,omitempty" norman:"type=reference[podSecurityPolicyTemplate]"` - DefaultClusterRoleForProjectMembers string `json:"defaultClusterRoleForProjectMembers,omitempty" norman:"type=reference[roleTemplate]"` + DisplayName string `json:"displayName"` + Description string `json:"description"` + Internal bool `json:"internal" norman:"nocreate,noupdate"` + DesiredAgentImage string `json:"desiredAgentImage"` + ImportedConfig *ImportedConfig `json:"importedConfig,omitempty" norman:"nocreate,noupdate"` + GoogleKubernetesEngineConfig *GoogleKubernetesEngineConfig `json:"googleKubernetesEngineConfig,omitempty"` + AzureKubernetesServiceConfig *AzureKubernetesServiceConfig `json:"azureKubernetesServiceConfig,omitempty"` + RancherKubernetesEngineConfig *RancherKubernetesEngineConfig `json:"rancherKubernetesEngineConfig,omitempty"` + AmazonElasticContainerServiceConfig *AmazonElasticContainerServiceConfig `json:"amazonElasticContainerServiceConfig,omitempty"` + DefaultPodSecurityPolicyTemplateName string `json:"defaultPodSecurityPolicyTemplateName,omitempty" norman:"type=reference[podSecurityPolicyTemplate]"` + DefaultClusterRoleForProjectMembers string `json:"defaultClusterRoleForProjectMembers,omitempty" norman:"type=reference[roleTemplate]"` } type ImportedConfig struct { @@ -193,6 +194,11 @@ type AzureKubernetesServiceConfig struct { ClientSecret string `json:"clientSecret,omitempty" norman:"required,type=password"` } +type AmazonElasticContainerServiceConfig struct { + AccessKey string `json:"accessKey" norman:"required"` + SecretKey string `json:"secretKey" norman:"required,type=password"` +} + type ClusterEvent struct { types.Namespaced v1.Event diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go index c94d543f..fc07dee8 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go @@ -43,11 +43,12 @@ var ( PluginsDocker: "plugins/docker", }, LoggingSystemImages: LoggingSystemImages{ - Fluentd: "rancher/fluentd:v0.1.4", - FluentdHelper: "rancher/fluentd-helper:v0.1.1", - Elaticsearch: "rancher/docker-elasticsearch-kubernetes:5.6.2", - Kibana: "kibana:5.6.4", - Busybox: "busybox", + Fluentd: "rancher/fluentd:v0.1.6", + FluentdHelper: "rancher/fluentd-helper:v0.1.2", + LogAggregatorFlexVolumeDriver: "rancher/log-aggregator:v0.1.2", + Elaticsearch: "rancher/docker-elasticsearch-kubernetes:5.6.2", + Kibana: "kibana:5.6.4", + Busybox: "busybox", }, } diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go index 1244b9a9..28ed1238 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go @@ -125,9 +125,10 @@ type SyslogConfig struct { } type LoggingSystemImages struct { - Fluentd string `json:"fluentd,omitempty"` - FluentdHelper string `json:"fluentdHelper,omitempty"` - Elaticsearch string `json:"elaticsearch,omitempty"` - Kibana string `json:"kibana,omitempty"` - Busybox string `json:"busybox,omitempty"` + Fluentd string `json:"fluentd,omitempty"` + FluentdHelper string `json:"fluentdHelper,omitempty"` + Elaticsearch string `json:"elaticsearch,omitempty"` + Kibana string `json:"kibana,omitempty"` + Busybox string `json:"busybox,omitempty"` + LogAggregatorFlexVolumeDriver string `json:"logAggregatorFlexVolumeDriver,omitempty"` } diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go index e9b2eb50..f248c332 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go @@ -72,6 +72,28 @@ type NodeStatus struct { NodeAnnotations map[string]string `json:"nodeAnnotations,omitempty"` NodeLabels map[string]string `json:"nodeLabels,omitempty"` NodeTaints []v1.Taint `json:"nodeTaints,omitempty"` + DockerInfo *DockerInfo `json:"dockerInfo,omitempty"` +} + +type DockerInfo struct { + ID string + Driver string + Debug bool + LoggingDriver string + CgroupDriver string + KernelVersion string + OperatingSystem string + OSType string + Architecture string + IndexServerAddress string + DockerRootDir string + HTTPProxy string + HTTPSProxy string + NoProxy string + Name string + Labels []string + ExperimentalBuild bool + ServerVersion string } var ( diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/rke_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/rke_types.go index 0103952d..a358d594 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/rke_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/rke_types.go @@ -250,6 +250,8 @@ type IngressConfig struct { Options map[string]string `yaml:"options" json:"options,omitempty"` // NodeSelector key pair NodeSelector map[string]string `yaml:"node_selector" json:"nodeSelector,omitempty"` + // Ingress controller extra arguments + ExtraArgs map[string]string `yaml:"extra_args" json:"extraArgs,omitempty"` } type RKEPlan struct { diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go index a8c56e1c..9fd314fb 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go @@ -48,6 +48,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*AlertSystemImages).DeepCopyInto(out.(*AlertSystemImages)) return nil }, InType: reflect.TypeOf(&AlertSystemImages{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*AmazonElasticContainerServiceConfig).DeepCopyInto(out.(*AmazonElasticContainerServiceConfig)) + return nil + }, InType: reflect.TypeOf(&AmazonElasticContainerServiceConfig{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*AuthAppInput).DeepCopyInto(out.(*AuthAppInput)) return nil @@ -248,6 +252,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*CustomConfig).DeepCopyInto(out.(*CustomConfig)) return nil }, InType: reflect.TypeOf(&CustomConfig{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*DockerInfo).DeepCopyInto(out.(*DockerInfo)) + return nil + }, InType: reflect.TypeOf(&DockerInfo{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*DynamicSchema).DeepCopyInto(out.(*DynamicSchema)) return nil @@ -1062,6 +1070,22 @@ func (in *AlertSystemImages) DeepCopy() *AlertSystemImages { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AmazonElasticContainerServiceConfig) DeepCopyInto(out *AmazonElasticContainerServiceConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AmazonElasticContainerServiceConfig. +func (in *AmazonElasticContainerServiceConfig) DeepCopy() *AmazonElasticContainerServiceConfig { + if in == nil { + return nil + } + out := new(AmazonElasticContainerServiceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AuthAppInput) DeepCopyInto(out *AuthAppInput) { *out = *in @@ -2231,6 +2255,15 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*in).DeepCopyInto(*out) } } + if in.AmazonElasticContainerServiceConfig != nil { + in, out := &in.AmazonElasticContainerServiceConfig, &out.AmazonElasticContainerServiceConfig + if *in == nil { + *out = nil + } else { + *out = new(AmazonElasticContainerServiceConfig) + **out = **in + } + } return } @@ -2404,6 +2437,27 @@ func (in *CustomConfig) DeepCopy() *CustomConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DockerInfo) DeepCopyInto(out *DockerInfo) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerInfo. +func (in *DockerInfo) DeepCopy() *DockerInfo { + if in == nil { + return nil + } + out := new(DockerInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DynamicSchema) DeepCopyInto(out *DynamicSchema) { *out = *in @@ -3162,6 +3216,13 @@ func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { (*out)[key] = val } } + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -4065,6 +4126,15 @@ func (in *NodeStatus) DeepCopyInto(out *NodeStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DockerInfo != nil { + in, out := &in.DockerInfo, &out.DockerInfo + if *in == nil { + *out = nil + } else { + *out = new(DockerInfo) + (*in).DeepCopyInto(*out) + } + } return }