From babe95f183624141d192f7c77c6611ae9d574c0b Mon Sep 17 00:00:00 2001 From: moelsayed Date: Mon, 6 Nov 2017 22:50:41 +0200 Subject: [PATCH] add network plugin support add template configration through env variables add kubedns addon --- cluster.yml | 2 + cluster/addons.go | 42 ++++++ cluster/cluster.go | 5 + cluster/network.go | 30 ++++ cmd/cluster.go | 10 ++ k8s/k8s.go | 56 ++++++++ package/kubectl/Dockerfile | 10 ++ package/kubectl/addons/kubedns.yaml | 202 +++++++++++++++++++++++++++ package/kubectl/entrypoint.sh | 22 +++ package/kubectl/network/flannel.yaml | 122 ++++++++++++++++ pki/constants.go | 1 + services/kubelet.go | 2 +- services/types.go | 1 + 13 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 cluster/addons.go create mode 100644 cluster/network.go create mode 100644 package/kubectl/Dockerfile create mode 100644 package/kubectl/addons/kubedns.yaml create mode 100755 package/kubectl/entrypoint.sh create mode 100644 package/kubectl/network/flannel.yaml diff --git a/cluster.yml b/cluster.yml index 2fbe710b..5060abee 100644 --- a/cluster.yml +++ b/cluster.yml @@ -1,4 +1,5 @@ --- +network_plugin: flannel hosts: - hostname: server1 ip: 1.1.1.1 @@ -27,6 +28,7 @@ services: kubelet: image: quay.io/coreos/hyperkube:v1.7.5_coreos.0 cluster_domain: cluster.local + cluster_dns_server: 10.233.0.3 infra_container_image: gcr.io/google_containers/pause-amd64:3.0 kubeproxy: image: quay.io/coreos/hyperkube:v1.7.5_coreos.0 diff --git a/cluster/addons.go b/cluster/addons.go new file mode 100644 index 00000000..148a3c73 --- /dev/null +++ b/cluster/addons.go @@ -0,0 +1,42 @@ +package cluster + +import ( + "fmt" + + "github.com/Sirupsen/logrus" + "github.com/rancher/rke/k8s" + "github.com/rancher/rke/pki" +) + +const ( + ClusterDnsServerIPEnvName = "RKE_DNS_SERVER" + ClusterDomainEnvName = "RKE_CLUSTER_DOMAIN" +) + +func (c *Cluster) DeployK8sAddOns() error { + if err := c.deployKubeDNS(); err != nil { + return err + } + return nil +} + +func (c *Cluster) deployKubeDNS() error { + logrus.Infof("[plugins] Setting up KubeDNS") + deployerHost := c.ControlPlaneHosts[0] + kubectlCmd := []string{"apply -f /addons/kubedns*.yaml"} + + env := []string{ + fmt.Sprintf("%s=%s", pki.KubeAdminConfigENVName, c.Certificates[pki.KubeAdminCommonName].Config), + fmt.Sprintf("%s=%s", ClusterDnsServerIPEnvName, c.ClusterDnsServer), + fmt.Sprintf("%s=%s", ClusterDomainEnvName, c.ClusterDomain), + } + + logrus.Infof("[plugins] Executing the deploy command..") + err := k8s.RunKubectlCmd(deployerHost.DClient, deployerHost.Hostname, kubectlCmd, env) + if err != nil { + return fmt.Errorf("Failed to run kubectl command: %v", err) + } + logrus.Infof("[plugins] kubeDNS deployed successfully..") + return nil + +} diff --git a/cluster/cluster.go b/cluster/cluster.go index 11e1daa0..f84950ad 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -22,6 +22,9 @@ type Cluster struct { KubernetesServiceIP net.IP Certificates map[string]pki.CertificatePKI ClusterDomain string + NetworkPlugin string `yaml:"network_plugin"` + ClusterCIDR string + ClusterDnsServer string } const ( @@ -65,6 +68,8 @@ func ParseConfig(clusterFile string) (*Cluster, error) { return nil, fmt.Errorf("Failed to get Kubernetes Service IP: %v", err) } c.ClusterDomain = c.Services.Kubelet.ClusterDomain + c.ClusterCIDR = c.Services.KubeController.ClusterCIDR + c.ClusterDnsServer = c.Services.Kubelet.ClusterDnsServer return c, nil } diff --git a/cluster/network.go b/cluster/network.go new file mode 100644 index 00000000..48b6a69c --- /dev/null +++ b/cluster/network.go @@ -0,0 +1,30 @@ +package cluster + +import ( + "fmt" + + "github.com/Sirupsen/logrus" + "github.com/rancher/rke/k8s" + "github.com/rancher/rke/pki" +) + +const ( + ClusterCIDREnvName = "RKE_CLUSTER_CIDR" +) + +func (c *Cluster) DeployNetworkPlugin() error { + logrus.Infof("[network] Setting up network plugin: %s", c.NetworkPlugin) + deployerHost := c.ControlPlaneHosts[0] + kubectlCmd := []string{"apply -f /network/" + c.NetworkPlugin + ".yaml"} + env := []string{ + fmt.Sprintf("%s=%s", pki.KubeAdminConfigENVName, c.Certificates[pki.KubeAdminCommonName].Config), + fmt.Sprintf("%s=%s", ClusterCIDREnvName, c.ClusterCIDR), + } + logrus.Infof("[network] Executing the deploy command..") + err := k8s.RunKubectlCmd(deployerHost.DClient, deployerHost.Hostname, kubectlCmd, env) + if err != nil { + return fmt.Errorf("Failed to run kubectl command: %v", err) + } + logrus.Infof("[network] Network plugin deployed successfully..") + return nil +} diff --git a/cmd/cluster.go b/cmd/cluster.go index 5fa94346..d3c4b4f7 100644 --- a/cmd/cluster.go +++ b/cmd/cluster.go @@ -77,6 +77,16 @@ func ClusterUp(clusterFile, authType string) (string, string, string, string, er return ApiURL, caCrt, clientCert, clientKey, err } + err = kubeCluster.DeployNetworkPlugin() + if err != nil { + return ApiURL, caCrt, clientCert, clientKey, err + } + + err = kubeCluster.DeployK8sAddOns() + if err != nil { + return ApiURL, caCrt, clientCert, clientKey, err + } + err = kubeCluster.SaveClusterState(clusterFile) if err != nil { return ApiURL, caCrt, clientCert, clientKey, err diff --git a/k8s/k8s.go b/k8s/k8s.go index 513caefe..274a02f6 100644 --- a/k8s/k8s.go +++ b/k8s/k8s.go @@ -1,6 +1,13 @@ package k8s import ( + "context" + "fmt" + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "github.com/rancher/rke/docker" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -8,6 +15,11 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const ( + KubectlImage = "melsayed/kubectl:latest" + KubctlContainer = "kubectl" +) + func NewClient(kubeConfigPath string) (*kubernetes.Clientset, error) { // use the current admin kubeconfig config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) @@ -88,3 +100,47 @@ func GetSecret(k8sClient *kubernetes.Clientset, secretName string) (*v1.Secret, func DeleteNode(k8sClient *kubernetes.Clientset, nodeName string) error { return k8sClient.Nodes().Delete(nodeName, &metav1.DeleteOptions{}) } + +func RunKubectlCmd(dClient *client.Client, hostname string, cmd []string, withEnv []string) error { + + logrus.Debugf("[kubectl] Using host [%s] for deployment", hostname) + logrus.Debugf("[kubectl] Pulling kubectl image..") + + if err := docker.PullImage(dClient, hostname, KubectlImage); err != nil { + return err + } + env := []string{} + if withEnv != nil { + env = append(env, withEnv...) + } + imageCfg := &container.Config{ + Image: KubectlImage, + Env: env, + Cmd: cmd, + } + logrus.Debugf("[kubectl] Creating kubectl container..") + resp, err := dClient.ContainerCreate(context.Background(), imageCfg, nil, nil, KubctlContainer) + if err != nil { + return fmt.Errorf("Failed to create kubectl container on host [%s]: %v", hostname, err) + } + logrus.Debugf("[kubectl] Container %s created..", resp.ID) + if err := dClient.ContainerStart(context.Background(), resp.ID, types.ContainerStartOptions{}); err != nil { + return fmt.Errorf("Failed to start kubectl container on host [%s]: %v", hostname, err) + } + logrus.Debugf("[kubectl] running command: %s", cmd) + statusCh, errCh := dClient.ContainerWait(context.Background(), resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("Failed to execute kubectl container on host [%s]: %v", hostname, err) + } + case status := <-statusCh: + if status.StatusCode != 0 { + return fmt.Errorf("kubectl command failed on host [%s]: exit status %v", hostname, status.StatusCode) + } + } + if err := dClient.ContainerRemove(context.Background(), resp.ID, types.ContainerRemoveOptions{}); err != nil { + return fmt.Errorf("Failed to remove kubectl container on host[%s]: %v", hostname, err) + } + return nil +} diff --git a/package/kubectl/Dockerfile b/package/kubectl/Dockerfile new file mode 100644 index 00000000..f48f4c37 --- /dev/null +++ b/package/kubectl/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.4 + +RUN apk update && apk add bash curl +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.7.5/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ + +RUN mkdir -p /network /addons /root/.kube/ +COPY network/*.yaml /network/ +COPY addons/*.yaml /addons/ +COPY entrypoint.sh /tmp/entrypoint.sh +ENTRYPOINT ["/tmp/entrypoint.sh"] diff --git a/package/kubectl/addons/kubedns.yaml b/package/kubectl/addons/kubedns.yaml new file mode 100644 index 00000000..cc9f6f21 --- /dev/null +++ b/package/kubectl/addons/kubedns.yaml @@ -0,0 +1,202 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. + +# Warning: This is a file generated from the base underscore template file: kubedns-controller.yaml.base +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + containers: + - name: kubedns + image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain=RKE_CLUSTER_DOMAIN. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config + - name: dnsmasq + image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5 + livenessProbe: + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --log-facility=- + - --server=/RKE_CLUSTER_DOMAIN/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5 + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.RKE_CLUSTER_DOMAIN,5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.RKE_CLUSTER_DOMAIN,5,A + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 20Mi + cpu: 10m + dnsPolicy: Default # Don't use cluster DNS. + serviceAccountName: kube-dns +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: RKE_DNS_SERVER + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/package/kubectl/entrypoint.sh b/package/kubectl/entrypoint.sh new file mode 100755 index 00000000..9948cc71 --- /dev/null +++ b/package/kubectl/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x + +# Set template configration +for i in $(env | grep -o RKE_.*=); do + key=$(echo "$i" | cut -f1 -d"=") + value=$(echo "${!key}") + for f in /network/*.yaml /addons/*.yaml; do + sed -i "s|${key}|${value}|g" ${f} + done +done + + +for i in $(env | grep -o KUBECFG_.*=); do + name="$(echo "$i" | cut -f1 -d"=" | tr '[:upper:]' '[:lower:]' | tr '_' '-').yaml" + env=$(echo "$i" | cut -f1 -d"=") + value=$(echo "${!env}") + if [ ! -f $SSL_CRTS_DIR/$name ]; then + echo "$value" > /root/.kube/config + fi +done + +kubectl ${@} diff --git a/package/kubectl/network/flannel.yaml b/package/kubectl/network/flannel.yaml new file mode 100644 index 00000000..740aebf0 --- /dev/null +++ b/package/kubectl/network/flannel.yaml @@ -0,0 +1,122 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: "kube-system" + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name":"cbr0", + "cniVersion":"0.3.1", + "plugins":[ + { + "type":"flannel", + "delegate":{ + "forceAddress":true, + "isDefaultGateway":true + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + } + } + ] + } + net-conf.json: | + { + "Network": "RKE_CLUSTER_CIDR", + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel + namespace: "kube-system" + labels: + tier: node + k8s-app: flannel +spec: + template: + metadata: + labels: + tier: node + k8s-app: flannel + spec: + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.8.0 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 300m + memory: 500M + requests: + cpu: 150m + memory: 64M + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: install-cni + image: quay.io/coreos/flannel-cni:v0.2.0 + command: ["/install-cni.sh"] + env: + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: kube-flannel-cfg + key: cni-conf.json + - name: CNI_CONF_NAME + value: "10-flannel.conflist" + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + - name: host-cni-bin + mountPath: /host/opt/cni/bin/ + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: host-cni-bin + hostPath: + path: /opt/cni/bin + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/pki/constants.go b/pki/constants.go index 99f6a44c..2e85b5e0 100644 --- a/pki/constants.go +++ b/pki/constants.go @@ -57,4 +57,5 @@ const ( KubeAdminCommonName = "kube-admin" KubeAdminOrganizationName = "system:masters" KubeAdminConfigPath = "admin.config" + KubeAdminConfigENVName = "KUBECFG_ADMIN" ) diff --git a/services/kubelet.go b/services/kubelet.go index 75d5cd39..b3942cf0 100644 --- a/services/kubelet.go +++ b/services/kubelet.go @@ -26,7 +26,7 @@ func buildKubeletConfig(host hosts.Host, kubeletService Kubelet, isMaster bool) "--cgroup-driver=cgroupfs", "--cgroups-per-qos=True", "--enforce-node-allocatable=", - "--cluster-dns=10.233.0.3", + "--cluster-dns=" + kubeletService.ClusterDnsServer, "--network-plugin=cni", "--cni-conf-dir=/etc/cni/net.d", "--cni-bin-dir=/opt/cni/bin", diff --git a/services/types.go b/services/types.go index 7984defc..b9730105 100644 --- a/services/types.go +++ b/services/types.go @@ -28,6 +28,7 @@ type Kubelet struct { Image string `yaml:"image"` ClusterDomain string `yaml:"cluster_domain"` InfraContainerImage string `yaml:"infra_container_image"` + ClusterDnsServer string `yaml:"cluster_dns_server"` } type Kubeproxy struct {