diff --git a/cluster.yml b/cluster.yml index bada98a3..11ceb4ce 100644 --- a/cluster.yml +++ b/cluster.yml @@ -28,6 +28,14 @@ authorization: mode: rbac options: + +# List of registry credentials, if you are using a Docker Hub registry, +# you can omit the `url` or set it to `docker.io` +private_registries: + - url: registry.com + user: Username + passowrd: password + nodes: - address: 1.1.1.1 user: ubuntu diff --git a/cluster/cluster.go b/cluster/cluster.go index ef428e42..7e33676c 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/rancher/rke/authz" + "github.com/rancher/rke/docker" "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" "github.com/rancher/rke/pki" @@ -35,6 +36,7 @@ type Cluster struct { ClusterDNSServer string DockerDialerFactory hosts.DialerFactory LocalConnDialerFactory hosts.DialerFactory + PrivateRegistriesMap map[string]v3.PrivateRegistry } const ( @@ -51,7 +53,7 @@ const ( func (c *Cluster) DeployControlPlane(ctx context.Context) error { // Deploy Etcd Plane - if err := services.RunEtcdPlane(ctx, c.EtcdHosts, c.Services.Etcd, c.LocalConnDialerFactory); err != nil { + if err := services.RunEtcdPlane(ctx, c.EtcdHosts, c.Services.Etcd, c.LocalConnDialerFactory, c.PrivateRegistriesMap); err != nil { return fmt.Errorf("[etcd] Failed to bring up Etcd Plane: %v", err) } // Deploy Control plane @@ -60,7 +62,8 @@ func (c *Cluster) DeployControlPlane(ctx context.Context) error { c.Services, c.SystemImages.KubernetesServicesSidecar, c.Authorization.Mode, - c.LocalConnDialerFactory); err != nil { + c.LocalConnDialerFactory, + c.PrivateRegistriesMap); err != nil { return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err) } // Apply Authz configuration after deploying controlplane @@ -78,7 +81,8 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context) error { c.Services, c.SystemImages.NginxProxy, c.SystemImages.KubernetesServicesSidecar, - c.LocalConnDialerFactory); err != nil { + c.LocalConnDialerFactory, + c.PrivateRegistriesMap); err != nil { return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err) } return nil @@ -105,6 +109,7 @@ func ParseCluster( ConfigPath: clusterFilePath, DockerDialerFactory: dockerDialerFactory, LocalConnDialerFactory: localConnDialerFactory, + PrivateRegistriesMap: make(map[string]v3.PrivateRegistry), } // Setting cluster Defaults c.setClusterDefaults(ctx) @@ -128,6 +133,14 @@ func ParseCluster( c.ConfigPath = DefaultClusterConfig } c.LocalKubeConfigPath = GetLocalKubeConfig(c.ConfigPath, configDir) + + for _, pr := range c.PrivateRegistries { + if pr.URL == "" { + pr.URL = docker.DockerRegistryURL + } + c.PrivateRegistriesMap[pr.URL] = pr + } + return c, nil } diff --git a/docker/docker.go b/docker/docker.go index e2bdede7..7ffb2c12 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -3,6 +3,8 @@ package docker import ( "archive/tar" "context" + "encoding/base64" + "encoding/json" "fmt" "io" "io/ioutil" @@ -10,24 +12,30 @@ import ( "reflect" "regexp" + ref "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/rancher/rke/log" + "github.com/rancher/types/apis/management.cattle.io/v3" "github.com/sirupsen/logrus" ) +const ( + DockerRegistryURL = "docker.io" +) + var K8sDockerVersions = map[string][]string{ "1.8": {"1.12.6", "1.13.1", "17.03.2"}, } -func DoRunContainer(ctx context.Context, dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName string, hostname string, plane string) error { +func DoRunContainer(ctx context.Context, dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName string, hostname string, plane string, prsMap map[string]v3.PrivateRegistry) error { container, err := dClient.ContainerInspect(ctx, containerName) if err != nil { if !client.IsErrNotFound(err) { return err } - if err := UseLocalOrPull(ctx, dClient, hostname, imageCfg.Image, plane); err != nil { + if err := UseLocalOrPull(ctx, dClient, hostname, imageCfg.Image, plane, prsMap); err != nil { return err } resp, err := dClient.ContainerCreate(ctx, imageCfg, hostCfg, nil, containerName) @@ -48,7 +56,7 @@ func DoRunContainer(ctx context.Context, dClient *client.Client, imageCfg *conta return err } if isUpgradable { - return DoRollingUpdateContainer(ctx, dClient, imageCfg, hostCfg, containerName, hostname, plane) + return DoRollingUpdateContainer(ctx, dClient, imageCfg, hostCfg, containerName, hostname, plane, prsMap) } return nil } @@ -62,7 +70,7 @@ func DoRunContainer(ctx context.Context, dClient *client.Client, imageCfg *conta return nil } -func DoRollingUpdateContainer(ctx context.Context, dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName, hostname, plane string) error { +func DoRollingUpdateContainer(ctx context.Context, dClient *client.Client, imageCfg *container.Config, hostCfg *container.HostConfig, containerName, hostname, plane string, prsMap map[string]v3.PrivateRegistry) error { logrus.Debugf("[%s] Checking for deployed [%s]", plane, containerName) isRunning, err := IsContainerRunning(ctx, dClient, hostname, containerName, false) if err != nil { @@ -72,7 +80,7 @@ func DoRollingUpdateContainer(ctx context.Context, dClient *client.Client, image logrus.Debugf("[%s] Container %s is not running on host [%s]", plane, containerName, hostname) return nil } - err = UseLocalOrPull(ctx, dClient, hostname, imageCfg.Image, plane) + err = UseLocalOrPull(ctx, dClient, hostname, imageCfg.Image, plane, prsMap) if err != nil { return err } @@ -150,8 +158,32 @@ func localImageExists(ctx context.Context, dClient *client.Client, hostname stri return true, nil } -func pullImage(ctx context.Context, dClient *client.Client, hostname string, containerImage string) error { - out, err := dClient.ImagePull(ctx, containerImage, types.ImagePullOptions{}) +func pullImage(ctx context.Context, dClient *client.Client, hostname string, containerImage string, prsMap map[string]v3.PrivateRegistry) error { + + pullOptions := types.ImagePullOptions{} + containerNamed, err := ref.ParseNormalizedNamed(containerImage) + if err != nil { + return err + } + + regURL := ref.Domain(containerNamed) + if pr, ok := prsMap[regURL]; ok { + // We do this if we have some docker.io login information + if pr.URL == DockerRegistryURL { + regAuth, err := getRegistryAuth(pr) + if err != nil { + return err + } + pullOptions.RegistryAuth = regAuth + } else { + // We have a registry, but it's not docker.io + // this could be public or private, ImagePull() can handle it + // if we provide a PrivilegeFunc + pullOptions.PrivilegeFunc = tryRegistryAuth(pr) + } + } + + out, err := dClient.ImagePull(ctx, containerImage, pullOptions) if err != nil { return fmt.Errorf("Can't pull Docker image [%s] for host [%s]: %v", containerImage, hostname, err) } @@ -165,7 +197,7 @@ func pullImage(ctx context.Context, dClient *client.Client, hostname string, con return nil } -func UseLocalOrPull(ctx context.Context, dClient *client.Client, hostname string, containerImage string, plane string) error { +func UseLocalOrPull(ctx context.Context, dClient *client.Client, hostname string, containerImage string, plane string, prsMap map[string]v3.PrivateRegistry) error { logrus.Debugf("[%s] Checking image [%s] on host [%s]", plane, containerImage, hostname) imageExists, err := localImageExists(ctx, dClient, hostname, containerImage) if err != nil { @@ -176,7 +208,7 @@ func UseLocalOrPull(ctx context.Context, dClient *client.Client, hostname string return nil } logrus.Debugf("[%s] Pulling image [%s] on host [%s]", plane, containerImage, hostname) - if err := pullImage(ctx, dClient, hostname, containerImage); err != nil { + if err := pullImage(ctx, dClient, hostname, containerImage, prsMap); err != nil { return err } log.Infof(ctx, "[%s] Successfully pulled image [%s] on host [%s]", plane, containerImage, hostname) @@ -302,3 +334,21 @@ func ReadContainerLogs(ctx context.Context, dClient *client.Client, containerNam return dClient.ContainerLogs(ctx, containerName, types.ContainerLogsOptions{ShowStdout: true}) } + +func tryRegistryAuth(pr v3.PrivateRegistry) types.RequestPrivilegeFunc { + return func() (string, error) { + return getRegistryAuth(pr) + } +} + +func getRegistryAuth(pr v3.PrivateRegistry) (string, error) { + authConfig := types.AuthConfig{ + Username: pr.User, + Password: pr.Password, + } + encodedJSON, err := json.Marshal(authConfig) + if err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(encodedJSON), nil +}