package services import ( "context" "fmt" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" v3 "github.com/rancher/rke/types" "github.com/rancher/rke/util" "github.com/sirupsen/logrus" ) const ( ETCDRole = "etcd" ControlRole = "controlplane" WorkerRole = "worker" SidekickServiceName = "sidekick" RBACAuthorizationMode = "rbac" KubeAPIContainerName = "kube-apiserver" KubeletContainerName = "kubelet" KubeproxyContainerName = "kube-proxy" KubeControllerContainerName = "kube-controller-manager" SchedulerContainerName = "kube-scheduler" EtcdContainerName = "etcd" EtcdSnapshotContainerName = "etcd-rolling-snapshots" EtcdSnapshotOnceContainerName = "etcd-snapshot-once" EtcdSnapshotRemoveContainerName = "etcd-remove-snapshot" EtcdRestoreContainerName = "etcd-restore" EtcdDownloadBackupContainerName = "etcd-download-backup" EtcdServeBackupContainerName = "etcd-Serve-backup" EtcdChecksumContainerName = "etcd-checksum-checker" EtcdStateFileContainerName = "etcd-extract-statefile" ControlPlaneConfigMapStateFileContainerName = "extract-statefile-configmap" NginxProxyContainerName = "nginx-proxy" SidekickContainerName = "service-sidekick" LogLinkContainerName = "rke-log-linker" LogCleanerContainerName = "rke-log-cleaner" KubeAPIPort = 6443 SchedulerPort = 10251 KubeControllerPort = 10252 KubeletPort = 10248 KubeproxyPort = 10256 WorkerThreads = util.WorkerThreads ContainerNameLabel = "io.rancher.rke.container.name" MCSLabel = "label=level:s0:c1000,c1001" ) type RestartFunc func(context.Context, *hosts.Host) error func runSidekick(ctx context.Context, host *hosts.Host, prsMap map[string]v3.PrivateRegistry, sidecarProcess v3.Process) error { isRunning, err := docker.IsContainerRunning(ctx, host.DClient, host.Address, SidekickContainerName, true) if err != nil { return err } imageCfg, hostCfg, _ := GetProcessConfig(sidecarProcess, host) isUpgradable := false if isRunning { isUpgradable, err = docker.IsContainerUpgradable(ctx, host.DClient, imageCfg, hostCfg, SidekickContainerName, host.Address, SidekickServiceName) if err != nil { return err } if !isUpgradable { log.Infof(ctx, "[%s] Sidekick container already created on host [%s]", SidekickServiceName, host.Address) return nil } } if err := docker.UseLocalOrPull(ctx, host.DClient, host.Address, sidecarProcess.Image, SidekickServiceName, prsMap); err != nil { return err } if isUpgradable { if err := docker.DoRemoveContainer(ctx, host.DClient, SidekickContainerName, host.Address); err != nil { return err } } if _, err := docker.CreateContainer(ctx, host.DClient, host.Address, SidekickContainerName, imageCfg, hostCfg); err != nil { return err } if host.DockerInfo.OSType == "windows" { // windows dockerfile VOLUME declaration must to satisfy one of them: // - a non-existing or empty directory // - a drive other than C: // so we could use a script to **start** the container to put expected resources into the "shared" directory, // like the action of `/usr/bin/sidecar.ps1` for windows rke-tools container return docker.StartContainer(ctx, host.DClient, host.Address, SidekickContainerName) } return nil } func removeSidekick(ctx context.Context, host *hosts.Host) error { return docker.DoRemoveContainer(ctx, host.DClient, SidekickContainerName, host.Address) } func GetProcessConfig(process v3.Process, host *hosts.Host) (*container.Config, *container.HostConfig, string) { imageCfg := &container.Config{ Entrypoint: process.Command, Cmd: process.Args, Env: process.Env, Image: process.Image, Labels: process.Labels, User: process.User, } // var pidMode container.PidMode // pidMode = process.PidMode _, portBindings, _ := nat.ParsePortSpecs(process.Publish) hostCfg := &container.HostConfig{ VolumesFrom: process.VolumesFrom, Binds: process.Binds, NetworkMode: container.NetworkMode(process.NetworkMode), PidMode: container.PidMode(process.PidMode), Privileged: process.Privileged, PortBindings: portBindings, } if len(process.RestartPolicy) > 0 { hostCfg.RestartPolicy = container.RestartPolicy{Name: process.RestartPolicy} } // The MCS label only needs to be applied when container is not running privileged, and running privileged negates need for applying the label // If Docker is configured with selinux-enabled:true, we need to specify MCS label to allow files from service-sidekick to be shared between containers if !process.Privileged && hosts.IsDockerSELinuxEnabled(host) { logrus.Debugf("Found selinux in DockerInfo.SecurityOptions on host [%s]", host.Address) // Check for containers having the sidekick container for _, volumeFrom := range hostCfg.VolumesFrom { if volumeFrom == SidekickContainerName { logrus.Debugf("Found [%s] in VolumesFrom on host [%s], applying MCSLabel [%s]", SidekickContainerName, host.Address, MCSLabel) hostCfg.SecurityOpt = []string{MCSLabel} } } // Check for sidekick container itself if value, ok := imageCfg.Labels[ContainerNameLabel]; ok { if value == SidekickContainerName { logrus.Debugf("Found [%s=%s] in Labels on host [%s], applying MCSLabel [%s]", ContainerNameLabel, SidekickContainerName, host.Address, MCSLabel) hostCfg.SecurityOpt = []string{MCSLabel} } } } return imageCfg, hostCfg, process.HealthCheck.URL } func GetHealthCheckURL(useTLS bool, port int) string { if useTLS { return fmt.Sprintf("%s%s:%d%s", HTTPSProtoPrefix, HealthzAddress, port, HealthzEndpoint) } return fmt.Sprintf("%s%s:%d%s", HTTPProtoPrefix, HealthzAddress, port, HealthzEndpoint) } func createLogLink(ctx context.Context, host *hosts.Host, containerName, plane, image string, prsMap map[string]v3.PrivateRegistry) error { logrus.Debugf("[%s] Creating log link for Container [%s] on host [%s]", plane, containerName, host.Address) containerInspect, err := docker.InspectContainer(ctx, host.DClient, host.Address, containerName) if err != nil { return err } containerID := containerInspect.ID containerLogPath := containerInspect.LogPath containerLogLink := fmt.Sprintf("%s/%s_%s.log", hosts.RKELogsPath, containerName, containerID) imageCfg := &container.Config{ Image: image, Tty: true, Cmd: []string{ "sh", "-c", fmt.Sprintf("mkdir -p %s ; ln -s %s %s", hosts.RKELogsPath, containerLogPath, containerLogLink), }, } hostCfg := &container.HostConfig{ Binds: []string{ "/var/lib:/var/lib", }, Privileged: true, } if err := docker.DoRemoveContainer(ctx, host.DClient, LogLinkContainerName, host.Address); err != nil { return err } if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, LogLinkContainerName, host.Address, plane, prsMap); err != nil { return err } if err := docker.DoRemoveContainer(ctx, host.DClient, LogLinkContainerName, host.Address); err != nil { return err } logrus.Debugf("[%s] Successfully created log link for Container [%s] on host [%s]", plane, containerName, host.Address) return nil }