diff --git a/cluster/addons.go b/cluster/addons.go index 9aed6ecb..f3ea2118 100644 --- a/cluster/addons.go +++ b/cluster/addons.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/Sirupsen/logrus" - "github.com/rancher/rke/k8s" - "github.com/rancher/rke/pki" ) const ( @@ -20,17 +18,12 @@ func (c *Cluster) DeployK8sAddOns() error { func (c *Cluster) deployKubeDNS() error { logrus.Infof("[plugins] Setting up KubeDNS") - deployerHost := c.ControlPlaneHosts[0] - kubectlCmd := []string{"apply -f /addons/kubedns*.yaml"} - env := []string{ - fmt.Sprintf("%s=%s", pki.KubeAdminConfigENVName, c.Certificates[pki.KubeAdminCommonName].Config), - fmt.Sprintf("%s=%s", ClusterDNSServerIPEnvName, c.ClusterDNSServer), - fmt.Sprintf("%s=%s", ClusterDomainEnvName, c.ClusterDomain), + kubectlCmd := &KubectlCommand{ + Cmd: []string{"apply -f /addons/kubedns*.yaml"}, } - logrus.Infof("[plugins] Executing the deploy command..") - err := k8s.RunKubectlCmd(deployerHost.DClient, deployerHost.Hostname, kubectlCmd, env) + err := c.RunKubectlCmd(kubectlCmd) if err != nil { return fmt.Errorf("Failed to run kubectl command: %v", err) } diff --git a/cluster/kubectl.go b/cluster/kubectl.go new file mode 100644 index 00000000..b8ecf2b1 --- /dev/null +++ b/cluster/kubectl.go @@ -0,0 +1,79 @@ +package cluster + +import ( + "context" + "fmt" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/rancher/rke/docker" + "github.com/rancher/rke/pki" +) + +const ( + KubectlImage = "melsayed/kubectl:latest" + KubctlContainer = "kubectl" +) + +type KubectlCommand struct { + Cmd []string + Env []string +} + +func (c *Cluster) buildClusterConfigEnv() []string { + // This needs to be updated when add more configuration + return []string{ + pki.ConvertConfigToENV(pki.KubeAdminConfigENVName, c.Certificates[pki.KubeAdminCommonName].Config), + pki.ConvertConfigToENV(ClusterCIDREnvName, c.ClusterCIDR), + pki.ConvertConfigToENV(ClusterDNSServerIPEnvName, c.ClusterDNSServer), + pki.ConvertConfigToENV(ClusterDomainEnvName, c.ClusterDomain), + } +} + +func (c *Cluster) RunKubectlCmd(kubectlCmd *KubectlCommand) error { + h := c.ControlPlaneHosts[0] + + logrus.Debugf("[kubectl] Using host [%s] for deployment", h.Hostname) + logrus.Debugf("[kubectl] Pulling kubectl image..") + + if err := docker.PullImage(h.DClient, h.Hostname, KubectlImage); err != nil { + return err + } + + clusterConfigEnv := c.buildClusterConfigEnv() + if kubectlCmd.Env != nil { + clusterConfigEnv = append(clusterConfigEnv, kubectlCmd.Env...) + } + + imageCfg := &container.Config{ + Image: KubectlImage, + Env: clusterConfigEnv, + Cmd: kubectlCmd.Cmd, + } + logrus.Debugf("[kubectl] Creating kubectl container..") + resp, err := h.DClient.ContainerCreate(context.Background(), imageCfg, nil, nil, KubctlContainer) + if err != nil { + return fmt.Errorf("Failed to create kubectl container on host [%s]: %v", h.Hostname, err) + } + logrus.Debugf("[kubectl] Container %s created..", resp.ID) + if err := h.DClient.ContainerStart(context.Background(), resp.ID, types.ContainerStartOptions{}); err != nil { + return fmt.Errorf("Failed to start kubectl container on host [%s]: %v", h.Hostname, err) + } + logrus.Debugf("[kubectl] running command: %s", kubectlCmd.Cmd) + statusCh, errCh := h.DClient.ContainerWait(context.Background(), resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("Failed to execute kubectl container on host [%s]: %v", h.Hostname, err) + } + case status := <-statusCh: + if status.StatusCode != 0 { + return fmt.Errorf("kubectl command failed on host [%s]: exit status %v", h.Hostname, status.StatusCode) + } + } + if err := h.DClient.ContainerRemove(context.Background(), resp.ID, types.ContainerRemoveOptions{}); err != nil { + return fmt.Errorf("Failed to remove kubectl container on host[%s]: %v", h.Hostname, err) + } + return nil +} diff --git a/cluster/network.go b/cluster/network.go index 48b6a69c..b5bb4e0e 100644 --- a/cluster/network.go +++ b/cluster/network.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/Sirupsen/logrus" - "github.com/rancher/rke/k8s" - "github.com/rancher/rke/pki" ) const ( @@ -14,14 +12,12 @@ const ( func (c *Cluster) DeployNetworkPlugin() error { logrus.Infof("[network] Setting up network plugin: %s", c.NetworkPlugin) - deployerHost := c.ControlPlaneHosts[0] - kubectlCmd := []string{"apply -f /network/" + c.NetworkPlugin + ".yaml"} - env := []string{ - fmt.Sprintf("%s=%s", pki.KubeAdminConfigENVName, c.Certificates[pki.KubeAdminCommonName].Config), - fmt.Sprintf("%s=%s", ClusterCIDREnvName, c.ClusterCIDR), + + kubectlCmd := &KubectlCommand{ + Cmd: []string{"apply -f /network/" + c.NetworkPlugin + ".yaml"}, } logrus.Infof("[network] Executing the deploy command..") - err := k8s.RunKubectlCmd(deployerHost.DClient, deployerHost.Hostname, kubectlCmd, env) + err := c.RunKubectlCmd(kubectlCmd) if err != nil { return fmt.Errorf("Failed to run kubectl command: %v", err) } diff --git a/k8s/k8s.go b/k8s/k8s.go index 274a02f6..513caefe 100644 --- a/k8s/k8s.go +++ b/k8s/k8s.go @@ -1,13 +1,6 @@ package k8s import ( - "context" - "fmt" - "github.com/Sirupsen/logrus" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - "github.com/rancher/rke/docker" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -15,11 +8,6 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -const ( - KubectlImage = "melsayed/kubectl:latest" - KubctlContainer = "kubectl" -) - func NewClient(kubeConfigPath string) (*kubernetes.Clientset, error) { // use the current admin kubeconfig config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) @@ -100,47 +88,3 @@ func GetSecret(k8sClient *kubernetes.Clientset, secretName string) (*v1.Secret, func DeleteNode(k8sClient *kubernetes.Clientset, nodeName string) error { return k8sClient.Nodes().Delete(nodeName, &metav1.DeleteOptions{}) } - -func RunKubectlCmd(dClient *client.Client, hostname string, cmd []string, withEnv []string) error { - - logrus.Debugf("[kubectl] Using host [%s] for deployment", hostname) - logrus.Debugf("[kubectl] Pulling kubectl image..") - - if err := docker.PullImage(dClient, hostname, KubectlImage); err != nil { - return err - } - env := []string{} - if withEnv != nil { - env = append(env, withEnv...) - } - imageCfg := &container.Config{ - Image: KubectlImage, - Env: env, - Cmd: cmd, - } - logrus.Debugf("[kubectl] Creating kubectl container..") - resp, err := dClient.ContainerCreate(context.Background(), imageCfg, nil, nil, KubctlContainer) - if err != nil { - return fmt.Errorf("Failed to create kubectl container on host [%s]: %v", hostname, err) - } - logrus.Debugf("[kubectl] Container %s created..", resp.ID) - if err := dClient.ContainerStart(context.Background(), resp.ID, types.ContainerStartOptions{}); err != nil { - return fmt.Errorf("Failed to start kubectl container on host [%s]: %v", hostname, err) - } - logrus.Debugf("[kubectl] running command: %s", cmd) - statusCh, errCh := dClient.ContainerWait(context.Background(), resp.ID, container.WaitConditionNotRunning) - select { - case err := <-errCh: - if err != nil { - return fmt.Errorf("Failed to execute kubectl container on host [%s]: %v", hostname, err) - } - case status := <-statusCh: - if status.StatusCode != 0 { - return fmt.Errorf("kubectl command failed on host [%s]: exit status %v", hostname, status.StatusCode) - } - } - if err := dClient.ContainerRemove(context.Background(), resp.ID, types.ContainerRemoveOptions{}); err != nil { - return fmt.Errorf("Failed to remove kubectl container on host[%s]: %v", hostname, err) - } - return nil -} diff --git a/pki/deploy.go b/pki/deploy.go index ae7db3c1..505c7587 100644 --- a/pki/deploy.go +++ b/pki/deploy.go @@ -16,38 +16,38 @@ import ( "k8s.io/client-go/util/cert" ) -func convertCrtToENV(name string, certificate *x509.Certificate) string { +func ConvertCrtToENV(name string, certificate *x509.Certificate) string { encodedCrt := cert.EncodeCertPEM(certificate) return fmt.Sprintf("%s=%s", name, string(encodedCrt)) } -func convertKeyToENV(name string, key *rsa.PrivateKey) string { +func ConvertKeyToENV(name string, key *rsa.PrivateKey) string { encodedKey := cert.EncodePrivateKeyPEM(key) return fmt.Sprintf("%s=%s", name, string(encodedKey)) } -func convertConfigToENV(name string, config string) string { +func ConvertConfigToENV(name string, config string) string { return fmt.Sprintf("%s=%s", name, config) } func DeployCertificatesOnMasters(cpHosts []hosts.Host, crtMap map[string]CertificatePKI) error { env := []string{ - convertCrtToENV(CACertENVName, crtMap[CACertName].Certificate), - convertKeyToENV(CAKeyENVName, crtMap[CACertName].Key), - convertCrtToENV(KubeAPICertENVName, crtMap[KubeAPICertName].Certificate), - convertKeyToENV(KubeAPIKeyENVName, crtMap[KubeAPICertName].Key), - convertCrtToENV(KubeControllerCertENVName, crtMap[KubeControllerName].Certificate), - convertKeyToENV(KubeControllerKeyENVName, crtMap[KubeControllerName].Key), - convertConfigToENV(KubeControllerConfigENVName, crtMap[KubeControllerName].Config), - convertCrtToENV(KubeSchedulerCertENVName, crtMap[KubeSchedulerName].Certificate), - convertKeyToENV(KubeSchedulerKeyENVName, crtMap[KubeSchedulerName].Key), - convertConfigToENV(KubeSchedulerConfigENVName, crtMap[KubeSchedulerName].Config), - convertCrtToENV(KubeProxyCertENVName, crtMap[KubeProxyName].Certificate), - convertKeyToENV(KubeProxyKeyENVName, crtMap[KubeProxyName].Key), - convertConfigToENV(KubeProxyConfigENVName, crtMap[KubeProxyName].Config), - convertCrtToENV(KubeNodeCertENVName, crtMap[KubeNodeName].Certificate), - convertKeyToENV(KubeNodeKeyENVName, crtMap[KubeNodeName].Key), - convertConfigToENV(KubeNodeConfigENVName, crtMap[KubeNodeName].Config), + ConvertCrtToENV(CACertENVName, crtMap[CACertName].Certificate), + ConvertKeyToENV(CAKeyENVName, crtMap[CACertName].Key), + ConvertCrtToENV(KubeAPICertENVName, crtMap[KubeAPICertName].Certificate), + ConvertKeyToENV(KubeAPIKeyENVName, crtMap[KubeAPICertName].Key), + ConvertCrtToENV(KubeControllerCertENVName, crtMap[KubeControllerName].Certificate), + ConvertKeyToENV(KubeControllerKeyENVName, crtMap[KubeControllerName].Key), + ConvertConfigToENV(KubeControllerConfigENVName, crtMap[KubeControllerName].Config), + ConvertCrtToENV(KubeSchedulerCertENVName, crtMap[KubeSchedulerName].Certificate), + ConvertKeyToENV(KubeSchedulerKeyENVName, crtMap[KubeSchedulerName].Key), + ConvertConfigToENV(KubeSchedulerConfigENVName, crtMap[KubeSchedulerName].Config), + ConvertCrtToENV(KubeProxyCertENVName, crtMap[KubeProxyName].Certificate), + ConvertKeyToENV(KubeProxyKeyENVName, crtMap[KubeProxyName].Key), + ConvertConfigToENV(KubeProxyConfigENVName, crtMap[KubeProxyName].Config), + ConvertCrtToENV(KubeNodeCertENVName, crtMap[KubeNodeName].Certificate), + ConvertKeyToENV(KubeNodeKeyENVName, crtMap[KubeNodeName].Key), + ConvertConfigToENV(KubeNodeConfigENVName, crtMap[KubeNodeName].Config), } for i := range cpHosts { err := doRunDeployer(&cpHosts[i], env) @@ -60,13 +60,13 @@ func DeployCertificatesOnMasters(cpHosts []hosts.Host, crtMap map[string]Certifi func DeployCertificatesOnWorkers(workerHosts []hosts.Host, crtMap map[string]CertificatePKI) error { env := []string{ - convertCrtToENV(CACertENVName, crtMap[CACertName].Certificate), - convertCrtToENV(KubeProxyCertENVName, crtMap[KubeProxyName].Certificate), - convertKeyToENV(KubeProxyKeyENVName, crtMap[KubeProxyName].Key), - convertConfigToENV(KubeProxyConfigENVName, crtMap[KubeProxyName].Config), - convertCrtToENV(KubeNodeCertENVName, crtMap[KubeNodeName].Certificate), - convertKeyToENV(KubeNodeKeyENVName, crtMap[KubeNodeName].Key), - convertConfigToENV(KubeNodeConfigENVName, crtMap[KubeNodeName].Config), + ConvertCrtToENV(CACertENVName, crtMap[CACertName].Certificate), + ConvertCrtToENV(KubeProxyCertENVName, crtMap[KubeProxyName].Certificate), + ConvertKeyToENV(KubeProxyKeyENVName, crtMap[KubeProxyName].Key), + ConvertConfigToENV(KubeProxyConfigENVName, crtMap[KubeProxyName].Config), + ConvertCrtToENV(KubeNodeCertENVName, crtMap[KubeNodeName].Certificate), + ConvertKeyToENV(KubeNodeKeyENVName, crtMap[KubeNodeName].Key), + ConvertConfigToENV(KubeNodeConfigENVName, crtMap[KubeNodeName].Config), } for i := range workerHosts { err := doRunDeployer(&workerHosts[i], env)