diff --git a/cluster/cluster.go b/cluster/cluster.go index 5429b1bb..d146fb48 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -45,6 +45,7 @@ const ( DNSMasqImage = "dnsmasq_image" KubeDNSSidecarImage = "kubedns_sidecar_image" KubeDNSAutoScalerImage = "kubedns_autoscaler_image" + ServiceSidekickImage = "service_sidekick_image" ) func (c *Cluster) DeployClusterPlanes() error { @@ -53,11 +54,18 @@ func (c *Cluster) DeployClusterPlanes() error { if err != nil { return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err) } - err = services.RunControlPlane(c.ControlPlaneHosts, c.EtcdHosts, c.Services) + err = services.RunControlPlane(c.ControlPlaneHosts, + c.EtcdHosts, + c.Services, + c.SystemImages[ServiceSidekickImage]) if err != nil { return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err) } - err = services.RunWorkerPlane(c.ControlPlaneHosts, c.WorkerHosts, c.Services, c.SystemImages[NginxProxyImage]) + err = services.RunWorkerPlane(c.ControlPlaneHosts, + c.WorkerHosts, + c.Services, + c.SystemImages[NginxProxyImage], + c.SystemImages[ServiceSidekickImage]) if err != nil { return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err) } @@ -157,6 +165,7 @@ func (c *Cluster) setClusterImageDefaults() { DNSMasqImage: DefaultDNSMasqImage, KubeDNSSidecarImage: DefaultKubeDNSSidecarImage, KubeDNSAutoScalerImage: DefaultKubeDNSAutoScalerImage, + ServiceSidekickImage: DefaultServiceSidekickImage, } for k, v := range systemImagesDefaultsMap { setDefaultIfEmptyMapValue(c.SystemImages, k, v) diff --git a/cluster/defaults.go b/cluster/defaults.go index 43579264..a317586f 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -13,10 +13,11 @@ const ( DefaultNetworkPlugin = "flannel" - DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0" - DefaultAplineImage = "alpine:latest" - DefaultNginxProxyImage = "rancher/rke-nginx-proxy:0.1.0" - DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.0" + DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0" + DefaultAplineImage = "alpine:latest" + DefaultNginxProxyImage = "rancher/rke-nginx-proxy:0.1.0" + DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.0" + DefaultServiceSidekickImage = "rancher/rke-service-sidekick:0.1.0" DefaultFlannelImage = "quay.io/coreos/flannel:v0.9.1" DefaultFlannelCNIImage = "quay.io/coreos/flannel-cni:v0.2.0" diff --git a/docker/docker.go b/docker/docker.go index 53c9363c..6d7071a1 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -20,7 +20,7 @@ var K8sDockerVersions = map[string][]string{ } func DoRunContainer(dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName string, hostname string, plane string) error { - isRunning, err := IsContainerRunning(dClient, hostname, containerName) + isRunning, err := IsContainerRunning(dClient, hostname, containerName, false) if err != nil { return err } @@ -56,7 +56,7 @@ func DoRunContainer(dClient *client.Client, imageCfg *container.Config, hostCfg func DoRollingUpdateContainer(dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName, hostname, plane string) error { logrus.Debugf("[%s] Checking for deployed [%s]", plane, containerName) - isRunning, err := IsContainerRunning(dClient, hostname, containerName) + isRunning, err := IsContainerRunning(dClient, hostname, containerName, false) if err != nil { return err } @@ -115,9 +115,9 @@ func DoRemoveContainer(dClient *client.Client, containerName, hostname string) e return nil } -func IsContainerRunning(dClient *client.Client, hostname string, containerName string) (bool, error) { +func IsContainerRunning(dClient *client.Client, hostname string, containerName string, all bool) (bool, error) { logrus.Debugf("Checking if container [%s] is running on host [%s]", containerName, hostname) - containers, err := dClient.ContainerList(context.Background(), types.ContainerListOptions{}) + containers, err := dClient.ContainerList(context.Background(), types.ContainerListOptions{All: all}) if err != nil { return false, fmt.Errorf("Can't get Docker containers for host [%s]: %v", hostname, err) diff --git a/package/service-sidekick/Dockerfile b/package/service-sidekick/Dockerfile new file mode 100644 index 00000000..1af4ff51 --- /dev/null +++ b/package/service-sidekick/Dockerfile @@ -0,0 +1,7 @@ +FROM scratch + +VOLUME /opt/rke + +COPY entrypoint.sh /opt/rke/entrypoint.sh + +CMD /bin/true diff --git a/package/service-sidekick/build-and-push.sh b/package/service-sidekick/build-and-push.sh new file mode 100755 index 00000000..285ab058 --- /dev/null +++ b/package/service-sidekick/build-and-push.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +ACCT=${ACCT:-rancher} + +docker build -t $ACCT/rke-service-sidekick:0.1.0 . +docker push $ACCT/rke-service-sidekick:0.1.0 diff --git a/package/service-sidekick/entrypoint.sh b/package/service-sidekick/entrypoint.sh new file mode 100755 index 00000000..a96e6e48 --- /dev/null +++ b/package/service-sidekick/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x + +if [ "$1" == "kubelet" ]; then + for i in $(DOCKER_API_VERSION=1.24 ./docker info 2>&1 | grep -i 'docker root dir' | cut -f2 -d:) /var/lib/docker /run /var/run; do + for m in $(tac /proc/mounts | awk '{print $2}' | grep ^${i}/); do + if [ "$m" != "/var/run/nscd" ] && [ "$m" != "/run/nscd" ]; then + umount $m || true + fi + done + done + mount --rbind /host/dev /dev + mount -o rw,remount /sys/fs/cgroup 2>/dev/null || true + for i in /sys/fs/cgroup/*; do + if [ -d $i ]; then + mkdir -p $i/kubepods + fi + done + CGROUPDRIVER=$(docker info | grep -i 'cgroup driver' | awk '{print $3}') + exec "$@" --cgroup-driver=$CGROUPDRIVER +fi + +exec "$@" diff --git a/pki/deploy.go b/pki/deploy.go index a8ea057d..5d0b448e 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -86,7 +86,7 @@ func doRunDeployer(host *hosts.Host, containerEnv []string, certDownloaderImage } logrus.Debugf("[certificates] Successfully started Certificate deployer container: %s", resp.ID) for { - isDeployerRunning, err := docker.IsContainerRunning(host.DClient, host.Address, CrtDownloaderContainer) + isDeployerRunning, err := docker.IsContainerRunning(host.DClient, host.Address, CrtDownloaderContainer, false) if err != nil { return err } diff --git a/services/controlplane.go b/services/controlplane.go index adb7653a..1219c45e 100644 --- a/services/controlplane.go +++ b/services/controlplane.go @@ -6,7 +6,7 @@ import ( "github.com/sirupsen/logrus" ) -func RunControlPlane(controlHosts []*hosts.Host, etcdHosts []*hosts.Host, controlServices v3.RKEConfigServices) error { +func RunControlPlane(controlHosts, etcdHosts []*hosts.Host, controlServices v3.RKEConfigServices, sidekickImage string) error { logrus.Infof("[%s] Building up Controller Plane..", ControlRole) for _, host := range controlHosts { @@ -15,6 +15,10 @@ func RunControlPlane(controlHosts []*hosts.Host, etcdHosts []*hosts.Host, contro return err } } + // run sidekick + if err := runSidekick(host, sidekickImage); err != nil { + return err + } // run kubeapi err := runKubeAPI(host, etcdHosts, controlServices.KubeAPI) if err != nil { @@ -66,6 +70,10 @@ func RemoveControlPlane(controlHosts []*hosts.Host, force bool) error { if err := removeKubeproxy(host); err != nil { return nil } + // remove Sidekick + if err := removeSidekick(host); err != nil { + return err + } } } logrus.Infof("[%s] Successfully teared down Controller Plane..", ControlRole) diff --git a/services/kubeapi.go b/services/kubeapi.go index dda12fc3..3c5e8aed 100644 --- a/services/kubeapi.go +++ b/services/kubeapi.go @@ -24,7 +24,8 @@ func removeKubeAPI(host *hosts.Host) error { func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcdConnString string) (*container.Config, *container.HostConfig) { imageCfg := &container.Config{ Image: kubeAPIService.Image, - Entrypoint: []string{"kube-apiserver", + Entrypoint: []string{"/opt/rke/entrypoint.sh", + "kube-apiserver", "--insecure-bind-address=127.0.0.1", "--bind-address=0.0.0.0", "--insecure-port=8080", @@ -44,6 +45,9 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd "--service-account-key-file=" + pki.KubeAPIKeyPath}, } hostCfg := &container.HostConfig{ + VolumesFrom: []string{ + SidekickContainerName, + }, Binds: []string{ "/etc/kubernetes:/etc/kubernetes", }, diff --git a/services/kubecontroller.go b/services/kubecontroller.go index 289b6912..35d68b61 100644 --- a/services/kubecontroller.go +++ b/services/kubecontroller.go @@ -22,7 +22,8 @@ func removeKubeController(host *hosts.Host) error { func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService) (*container.Config, *container.HostConfig) { imageCfg := &container.Config{ Image: kubeControllerService.Image, - Entrypoint: []string{"kube-controller-manager", + Entrypoint: []string{"/opt/rke/entrypoint.sh", + "kube-controller-manager", "--address=0.0.0.0", "--cloud-provider=", "--leader-elect=true", @@ -39,6 +40,9 @@ func buildKubeControllerConfig(kubeControllerService v3.KubeControllerService) ( }, } hostCfg := &container.HostConfig{ + VolumesFrom: []string{ + SidekickContainerName, + }, Binds: []string{ "/etc/kubernetes:/etc/kubernetes", }, diff --git a/services/kubelet.go b/services/kubelet.go index 22b36d90..1291134a 100644 --- a/services/kubelet.go +++ b/services/kubelet.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/docker/docker/api/types/container" - "github.com/docker/go-connections/nat" "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" "github.com/rancher/rke/pki" @@ -23,12 +22,12 @@ func removeKubelet(host *hosts.Host) error { func buildKubeletConfig(host *hosts.Host, kubeletService v3.KubeletService) (*container.Config, *container.HostConfig) { imageCfg := &container.Config{ Image: kubeletService.Image, - Entrypoint: []string{"kubelet", + Entrypoint: []string{"/opt/rke/entrypoint.sh", + "kubelet", "--v=2", "--address=0.0.0.0", "--cluster-domain=" + kubeletService.ClusterDomain, "--pod-infra-container-image=" + kubeletService.InfraContainerImage, - "--cgroup-driver=cgroupfs", "--cgroups-per-qos=True", "--enforce-node-allocatable=", "--hostname-override=" + host.HostnameOverride, @@ -54,30 +53,24 @@ func buildKubeletConfig(host *hosts.Host, kubeletService v3.KubeletService) (*co } } hostCfg := &container.HostConfig{ + VolumesFrom: []string{ + SidekickContainerName, + }, Binds: []string{ "/etc/kubernetes:/etc/kubernetes", "/etc/cni:/etc/cni:ro", "/opt/cni:/opt/cni:ro", "/etc/resolv.conf:/etc/resolv.conf", - "/sys:/sys:ro", + "/sys:/sys", "/var/lib/docker:/var/lib/docker:rw", "/var/lib/kubelet:/var/lib/kubelet:shared", "/var/run:/var/run:rw", "/run:/run", - "/dev:/host/dev", - "/sys/fs/cgroup:/sys/fs/cgroup:rw"}, + "/dev:/host/dev"}, NetworkMode: "host", PidMode: "host", Privileged: true, RestartPolicy: container.RestartPolicy{Name: "always"}, - PortBindings: nat.PortMap{ - "8080/tcp": []nat.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: "8080", - }, - }, - }, } for arg, value := range kubeletService.ExtraArgs { cmd := fmt.Sprintf("--%s=%s", arg, value) diff --git a/services/kubeproxy.go b/services/kubeproxy.go index bc845936..db0a716f 100644 --- a/services/kubeproxy.go +++ b/services/kubeproxy.go @@ -22,13 +22,17 @@ func removeKubeproxy(host *hosts.Host) error { func buildKubeproxyConfig(host *hosts.Host, kubeproxyService v3.KubeproxyService) (*container.Config, *container.HostConfig) { imageCfg := &container.Config{ Image: kubeproxyService.Image, - Entrypoint: []string{"kube-proxy", + Entrypoint: []string{"/opt/rke/entrypoint.sh", + "kube-proxy", "--v=2", "--healthz-bind-address=0.0.0.0", "--kubeconfig=" + pki.KubeProxyConfigPath, }, } hostCfg := &container.HostConfig{ + VolumesFrom: []string{ + SidekickContainerName, + }, Binds: []string{ "/etc/kubernetes:/etc/kubernetes", }, diff --git a/services/scheduler.go b/services/scheduler.go index c8b374ba..aab14d74 100644 --- a/services/scheduler.go +++ b/services/scheduler.go @@ -22,7 +22,8 @@ func removeScheduler(host *hosts.Host) error { func buildSchedulerConfig(host *hosts.Host, schedulerService v3.SchedulerService) (*container.Config, *container.HostConfig) { imageCfg := &container.Config{ Image: schedulerService.Image, - Entrypoint: []string{"kube-scheduler", + Entrypoint: []string{"/opt/rke/entrypoint.sh", + "kube-scheduler", "--leader-elect=true", "--v=2", "--address=0.0.0.0", @@ -30,6 +31,9 @@ func buildSchedulerConfig(host *hosts.Host, schedulerService v3.SchedulerService }, } hostCfg := &container.HostConfig{ + VolumesFrom: []string{ + SidekickContainerName, + }, Binds: []string{ "/etc/kubernetes:/etc/kubernetes", }, diff --git a/services/services.go b/services/services.go index 0bfcb078..7649fb6c 100644 --- a/services/services.go +++ b/services/services.go @@ -3,6 +3,11 @@ package services import ( "fmt" "net" + + "github.com/docker/docker/api/types/container" + "github.com/rancher/rke/docker" + "github.com/rancher/rke/hosts" + "github.com/sirupsen/logrus" ) const ( @@ -17,6 +22,7 @@ const ( SchedulerContainerName = "scheduler" EtcdContainerName = "etcd" NginxProxyContainerName = "nginx-proxy" + SidekickContainerName = "service-sidekick" ) func GetKubernetesServiceIP(serviceClusterRange string) (net.IP, error) { @@ -33,3 +39,36 @@ func GetKubernetesServiceIP(serviceClusterRange string) (net.IP, error) { } return ip, nil } + +func buildSidekickConfig(sidekickImage string) (*container.Config, *container.HostConfig) { + imageCfg := &container.Config{ + Image: sidekickImage, + } + hostCfg := &container.HostConfig{ + NetworkMode: "none", + } + return imageCfg, hostCfg +} + +func runSidekick(host *hosts.Host, sidekickImage string) error { + isRunning, err := docker.IsContainerRunning(host.DClient, host.Address, SidekickContainerName, true) + if err != nil { + return err + } + if isRunning { + logrus.Infof("[sidekick] Sidekick container already created on host [%s]", host.Address) + return nil + } + imageCfg, hostCfg := buildSidekickConfig(sidekickImage) + if err := docker.PullImage(host.DClient, host.Address, sidekickImage); err != nil { + return err + } + if _, err := docker.CreateContiner(host.DClient, host.Address, SidekickContainerName, imageCfg, hostCfg); err != nil { + return err + } + return nil +} + +func removeSidekick(host *hosts.Host) error { + return docker.DoRemoveContainer(host.DClient, SidekickContainerName, host.Address) +} diff --git a/services/workerplane.go b/services/workerplane.go index d99483e5..39f1f43c 100644 --- a/services/workerplane.go +++ b/services/workerplane.go @@ -6,9 +6,14 @@ import ( "github.com/sirupsen/logrus" ) -func RunWorkerPlane(controlHosts []*hosts.Host, workerHosts []*hosts.Host, workerServices v3.RKEConfigServices, nginxProxyImage string) error { +func RunWorkerPlane(controlHosts, workerHosts []*hosts.Host, workerServices v3.RKEConfigServices, nginxProxyImage, sidekickImage string) error { logrus.Infof("[%s] Building up Worker Plane..", WorkerRole) for _, host := range controlHosts { + // run sidekick + if err := runSidekick(host, sidekickImage); err != nil { + return err + } + // run kubelet // only one master for now if err := runKubelet(host, workerServices.Kubelet); err != nil { return err @@ -24,6 +29,10 @@ func RunWorkerPlane(controlHosts []*hosts.Host, workerHosts []*hosts.Host, worke return err } } + // run sidekick + if err := runSidekick(host, sidekickImage); err != nil { + return err + } // run kubelet if err := runKubelet(host, workerServices.Kubelet); err != nil { return err @@ -55,6 +64,9 @@ func RemoveWorkerPlane(workerHosts []*hosts.Host, force bool) error { if err := removeNginxProxy(host); err != nil { return err } + if err := removeSidekick(host); err != nil { + return err + } logrus.Infof("[%s] Successfully teared down Worker Plane..", WorkerRole) }