Merge pull request #35594 from madhusudancs/federation-kubefed-init-00

Automatic merge from submit-queue

[Federation][init] Implement `kubefed init` command that performs federation control plane bootstrap.

Please review only the last commit here. This is based on PR #35593 which will be reviewed independently.

I am intentionally not including the unit tests in this PR to better distribute and parallelize reviews. This PR is already big.

I will add a release note separately for this entire feature, so please don't worry too much about the release note here in the PR.

Design Doc: PR #34484

cc @kubernetes/sig-cluster-federation @quinton-hoole @nikhiljindal
This commit is contained in:
Kubernetes Submit Queue 2016-11-03 02:08:14 -07:00 committed by GitHub
commit 2f84a7c8d2
8 changed files with 567 additions and 13 deletions

View File

@ -20,6 +20,7 @@ go_library(
tags = ["automanaged"],
deps = [
"//federation/apis/federation:go_default_library",
"//federation/pkg/kubefed/init:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/errors:go_default_library",

View File

@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
"go_test",
"cgo_library",
)
go_library(
name = "go_default_library",
srcs = ["init.go"],
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/util:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/resource:go_default_library",
"//pkg/apis/extensions:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/kubectl/cmd/templates:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/util/cert:go_default_library",
"//pkg/util/cert/triple:go_default_library",
"//pkg/util/intstr:go_default_library",
"//pkg/util/wait:go_default_library",
"//vendor:github.com/spf13/cobra",
],
)

View File

@ -0,0 +1,517 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// TODO(madhusdancs):
// 1. Add a dry-run support.
// 2. Make all the API object names customizable.
// Ex: federation-apiserver, federation-controller-manager, etc.
// 3. Make image name and tag customizable.
// 4. Separate etcd container from API server pod as a first step towards enabling HA.
// 5. Generate credentials of the following types for the API server:
// i. "known_tokens.csv"
// ii. "basic_auth.csv"
// 6. Add the ability to customize DNS domain suffix. It should probably be derived
// from cluster config.
// 7. Make etcd PVC size configurable.
package init
import (
"fmt"
"io"
"time"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/apis/extensions"
client "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
certutil "k8s.io/kubernetes/pkg/util/cert"
triple "k8s.io/kubernetes/pkg/util/cert/triple"
"k8s.io/kubernetes/pkg/util/intstr"
"k8s.io/kubernetes/pkg/util/wait"
"github.com/spf13/cobra"
)
const (
APIServerCN = "federation-apiserver"
ControllerManagerCN = "federation-controller-manager"
HostClusterLocalDNSZoneName = "cluster.local."
lbAddrRetryInterval = 5 * time.Second
)
var (
init_long = templates.LongDesc(`
Initialize a federation control plane.
Federation control plane is hosted inside a Kubernetes
cluster. The host cluster must be specified using the
--host-cluster-context flag.`)
init_example = templates.Examples(`
# Initialize federation control plane for a federation
# named foo in the host cluster whose local kubeconfig
# context is bar.
kubectl init foo --host-cluster-context=bar`)
componentLabel = map[string]string{
"app": "federated-cluster",
}
apiserverSvcSelector = map[string]string{
"app": "federated-cluster",
"module": "federation-apiserver",
}
apiserverPodLabels = map[string]string{
"app": "federated-cluster",
"module": "federation-apiserver",
}
controllerManagerPodLabels = map[string]string{
"app": "federated-cluster",
"module": "federation-controller-manager",
}
hyperkubeImage = "gcr.io/google_containers/hyperkube-amd64:v1.5.0"
)
// NewCmdInit defines the `init` command that bootstraps a federation
// control plane inside a set of host clusters.
func NewCmdInit(f cmdutil.Factory, cmdOut io.Writer, config util.AdminConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "init FEDERATION_NAME --host-cluster-context=HOST_CONTEXT",
Short: "init initializes a federation control plane",
Long: init_long,
Example: init_example,
Run: func(cmd *cobra.Command, args []string) {
err := initFederation(f, cmdOut, config, cmd, args)
cmdutil.CheckErr(err)
},
}
util.AddSubcommandFlags(cmd)
cmd.Flags().String("dns-zone-name", "", "DNS suffix for this federation. Federated Service DNS names are published with this suffix.")
return cmd
}
type entityKeyPairs struct {
ca *triple.KeyPair
server *triple.KeyPair
controllerManager *triple.KeyPair
}
// initFederation initializes a federation control plane.
// See the design doc in https://github.com/kubernetes/kubernetes/pull/34484
// for details.
func initFederation(f cmdutil.Factory, cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Command, args []string) error {
initFlags, err := util.GetSubcommandFlags(cmd, args)
if err != nil {
return err
}
dnsZoneName := cmdutil.GetFlagString(cmd, "dns-zone-name")
hostFactory := config.HostFactory(initFlags.Host, initFlags.Kubeconfig)
hostClientset, err := hostFactory.ClientSet()
if err != nil {
return err
}
serverName := fmt.Sprintf("%s-apiserver", initFlags.Name)
serverCredName := fmt.Sprintf("%s-credentials", serverName)
cmName := fmt.Sprintf("%s-controller-manager", initFlags.Name)
cmKubeconfigName := fmt.Sprintf("%s-kubeconfig", cmName)
// 1. Create a namespace for federation system components
_, err = createNamespace(hostClientset, initFlags.FederationSystemNamespace)
if err != nil {
return err
}
// 2. Expose a network endpoint for the federation API server
svc, err := createService(hostClientset, initFlags.FederationSystemNamespace, serverName)
if err != nil {
return err
}
ips, hostnames, err := waitForLoadBalancerAddress(hostClientset, svc)
if err != nil {
return err
}
// 3. Generate TLS certificates and credentials
entKeyPairs, err := genCerts(initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, HostClusterLocalDNSZoneName, ips, hostnames)
if err != nil {
return err
}
_, err = createAPIServerCredentialsSecret(hostClientset, initFlags.FederationSystemNamespace, serverCredName, entKeyPairs)
if err != nil {
return err
}
// 4. Create a kubeconfig secret
_, err = createControllerManagerKubeconfigSecret(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmKubeconfigName, entKeyPairs)
if err != nil {
return err
}
// 5. Create a persistent volume and a claim to store the federation
// API server's state. This is where federation API server's etcd
// stores its data.
pvc, err := createPVC(hostClientset, initFlags.FederationSystemNamespace, svc.Name)
if err != nil {
return err
}
// Since only one IP address can be specified as advertise address,
// we arbitrarily pick the first availabe IP address
advertiseAddress := ""
if len(ips) > 0 {
advertiseAddress = ips[0]
}
// 6. Create federation API server
_, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, serverCredName, pvc.Name, advertiseAddress)
if err != nil {
return err
}
// 7. Create federation controller manager
_, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, cmName, cmKubeconfigName, dnsZoneName)
if err != nil {
return err
}
return nil
}
func createNamespace(clientset *client.Clientset, namespace string) (*api.Namespace, error) {
ns := &api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: namespace,
},
}
return clientset.Core().Namespaces().Create(ns)
}
func createService(clientset *client.Clientset, namespace, svcName string) (*api.Service, error) {
svc := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: svcName,
Namespace: namespace,
Labels: componentLabel,
},
Spec: api.ServiceSpec{
Type: api.ServiceTypeLoadBalancer,
Selector: apiserverSvcSelector,
Ports: []api.ServicePort{
{
Name: "https",
Protocol: "TCP",
Port: 443,
TargetPort: intstr.FromInt(443),
},
},
},
}
return clientset.Core().Services(namespace).Create(svc)
}
func waitForLoadBalancerAddress(clientset *client.Clientset, svc *api.Service) ([]string, []string, error) {
ips := []string{}
hostnames := []string{}
err := wait.PollInfinite(lbAddrRetryInterval, func() (bool, error) {
pollSvc, err := clientset.Core().Services(svc.Namespace).Get(svc.Name)
if err != nil {
return false, nil
}
if ings := pollSvc.Status.LoadBalancer.Ingress; len(ings) > 0 {
for _, ing := range ings {
if len(ing.IP) > 0 {
ips = append(ips, ing.IP)
}
if len(ing.Hostname) > 0 {
hostnames = append(hostnames, ing.Hostname)
}
}
if len(ips) > 0 || len(hostnames) > 0 {
return true, nil
}
}
return false, nil
})
if err != nil {
return nil, nil, err
}
return ips, hostnames, nil
}
func genCerts(svcNamespace, name, svcName, localDNSZoneName string, ips, hostnames []string) (*entityKeyPairs, error) {
ca, err := triple.NewCA(name)
if err != nil {
return nil, fmt.Errorf("failed to create CA key and certificate: %v", err)
}
server, err := triple.NewServerKeyPair(ca, APIServerCN, svcName, svcNamespace, localDNSZoneName, ips, hostnames)
if err != nil {
return nil, fmt.Errorf("failed to create federation API server key and certificate: %v", err)
}
cm, err := triple.NewClientKeyPair(ca, ControllerManagerCN)
if err != nil {
return nil, fmt.Errorf("failed to create federation controller manager client key and certificate: %v", err)
}
return &entityKeyPairs{
ca: ca,
server: server,
controllerManager: cm,
}, nil
}
func createAPIServerCredentialsSecret(clientset *client.Clientset, namespace, credentialsName string, entKeyPairs *entityKeyPairs) (*api.Secret, error) {
// Build the secret object with API server credentials.
secret := &api.Secret{
ObjectMeta: api.ObjectMeta{
Name: credentialsName,
Namespace: namespace,
},
Data: map[string][]byte{
"ca.crt": certutil.EncodeCertPEM(entKeyPairs.ca.Cert),
"server.crt": certutil.EncodeCertPEM(entKeyPairs.server.Cert),
"server.key": certutil.EncodePrivateKeyPEM(entKeyPairs.server.Key),
},
}
// Boilerplate to create the secret in the host cluster.
return clientset.Core().Secrets(namespace).Create(secret)
}
func createControllerManagerKubeconfigSecret(clientset *client.Clientset, namespace, name, svcName, kubeconfigName string, entKeyPairs *entityKeyPairs) (*api.Secret, error) {
basicClientConfig := kubeadmutil.CreateBasicClientConfig(
name,
fmt.Sprintf("https://%s", svcName),
certutil.EncodeCertPEM(entKeyPairs.ca.Cert),
)
config := kubeadmutil.MakeClientConfigWithCerts(
basicClientConfig,
name,
"federation-controller-manager",
certutil.EncodePrivateKeyPEM(entKeyPairs.controllerManager.Key),
certutil.EncodeCertPEM(entKeyPairs.controllerManager.Cert),
)
return util.CreateKubeconfigSecret(clientset, config, namespace, kubeconfigName, false)
}
func createPVC(clientset *client.Clientset, namespace, svcName string) (*api.PersistentVolumeClaim, error) {
capacity, err := resource.ParseQuantity("10Gi")
if err != nil {
return nil, err
}
pvc := &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("%s-etcd-claim", svcName),
Namespace: namespace,
Labels: componentLabel,
Annotations: map[string]string{
"volume.alpha.kubernetes.io/storage-class": "yes",
},
},
Spec: api.PersistentVolumeClaimSpec{
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceStorage: capacity,
},
},
},
}
return clientset.Core().PersistentVolumeClaims(namespace).Create(pvc)
}
func createAPIServer(clientset *client.Clientset, namespace, name, credentialsName, pvcName, advertiseAddress string) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-apiserver",
"--bind-address=0.0.0.0",
"--etcd-servers=http://localhost:2379",
"--service-cluster-ip-range=10.0.0.0/16",
"--secure-port=443",
"--token-auth-file=/etc/federation/apiserver/known_tokens.csv",
"--basic-auth-file=/etc/federation/apiserver/basic_auth.csv",
"--client-ca-file=/etc/federation/apiserver/ca.crt",
"--tls-cert-file=/etc/federation/apiserver/server.crt",
"--tls-private-key-file=/etc/federation/apiserver/server.key",
}
if advertiseAddress != "" {
command = append(command, fmt.Sprintf("--advertise-address=%s", advertiseAddress))
}
dataVolumeName := "etcddata"
dep := &extensions.Deployment{
ObjectMeta: api.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: componentLabel,
},
Spec: extensions.DeploymentSpec{
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: name,
Labels: apiserverPodLabels,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "apiserver",
Image: hyperkubeImage,
Command: command,
Ports: []api.ContainerPort{
{
Name: "https",
ContainerPort: 443,
},
{
Name: "local",
ContainerPort: 8080,
},
},
VolumeMounts: []api.VolumeMount{
{
Name: credentialsName,
MountPath: "/etc/federation/apiserver",
ReadOnly: true,
},
},
},
{
Name: "etcd",
Image: "quay.io/coreos/etcd:v2.3.3",
Command: []string{
"/etcd",
"--data-dir",
"/var/etcd/data",
},
VolumeMounts: []api.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/var/etcd",
},
},
},
},
Volumes: []api.Volume{
{
Name: credentialsName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: credentialsName,
},
},
},
{
Name: dataVolumeName,
VolumeSource: api.VolumeSource{
PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
},
},
},
},
},
},
},
}
return clientset.Extensions().Deployments(namespace).Create(dep)
}
func createControllerManager(clientset *client.Clientset, namespace, name, kubeconfigName, dnsZoneName string) (*extensions.Deployment, error) {
dep := &extensions.Deployment{
ObjectMeta: api.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: componentLabel,
},
Spec: extensions.DeploymentSpec{
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: name,
Labels: controllerManagerPodLabels,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "controller-manager",
Image: hyperkubeImage,
Command: []string{
"/hyperkube",
"federation-controller-manager",
"--master=https://federation-apiserver",
"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
"--dns-provider=gce",
"--dns-provider-config=",
fmt.Sprintf("--federation-name=%s", name),
fmt.Sprintf("--zone-name=%s", dnsZoneName),
},
VolumeMounts: []api.VolumeMount{
{
Name: kubeconfigName,
MountPath: "/etc/federation/controller-manager",
ReadOnly: true,
},
},
Env: []api.EnvVar{
{
Name: "POD_NAMESPACE",
ValueFrom: &api.EnvVarSource{
FieldRef: &api.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
},
},
},
Volumes: []api.Volume{
{
Name: kubeconfigName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: kubeconfigName,
},
},
},
},
},
},
},
}
return clientset.Extensions().Deployments(namespace).Create(dep)
}

View File

@ -84,7 +84,7 @@ func joinFederation(f cmdutil.Factory, cmdOut io.Writer, config util.AdminConfig
}
dryRun := cmdutil.GetDryRunFlag(cmd)
glog.V(2).Infof("Args and flags: name %s, host: %s, host-system-namespace: %s, kubeconfig: %s, dry-run: %s", joinFlags.Name, joinFlags.Host, joinFlags.HostSystemNamespace, joinFlags.Kubeconfig, dryRun)
glog.V(2).Infof("Args and flags: name %s, host: %s, host-system-namespace: %s, kubeconfig: %s, dry-run: %s", joinFlags.Name, joinFlags.Host, joinFlags.FederationSystemNamespace, joinFlags.Kubeconfig, dryRun)
po := config.PathOptions()
po.LoadingRules.ExplicitPath = joinFlags.Kubeconfig
@ -116,7 +116,7 @@ func joinFederation(f cmdutil.Factory, cmdOut io.Writer, config util.AdminConfig
// don't have to print the created secret in the default case.
// Having said that, secret generation machinery could be altered to
// suit our needs, but it is far less invasive and readable this way.
_, err = createSecret(hostFactory, clientConfig, joinFlags.HostSystemNamespace, joinFlags.Name, dryRun)
_, err = createSecret(hostFactory, clientConfig, joinFlags.FederationSystemNamespace, joinFlags.Name, dryRun)
if err != nil {
glog.V(2).Infof("Failed creating the cluster credentials secret: %v", err)
return err

View File

@ -19,6 +19,7 @@ package kubefed
import (
"io"
kubefedinit "k8s.io/kubernetes/federation/pkg/kubefed/init"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
kubectl "k8s.io/kubernetes/pkg/kubectl/cmd"
@ -52,6 +53,7 @@ func NewKubeFedCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob
{
Message: "Basic Commands:",
Commands: []*cobra.Command{
kubefedinit.NewCmdInit(f, out, util.NewAdminConfig(clientcmd.NewDefaultPathOptions())),
NewCmdJoin(f, out, util.NewAdminConfig(clientcmd.NewDefaultPathOptions())),
NewCmdUnjoin(f, out, err, util.NewAdminConfig(clientcmd.NewDefaultPathOptions())),
},

View File

@ -84,7 +84,7 @@ func unjoinFederation(f cmdutil.Factory, cmdOut, cmdErr io.Writer, config util.A
// We want a separate client factory to communicate with the
// federation host cluster. See join_federation.go for details.
hostFactory := config.HostFactory(unjoinFlags.Host, unjoinFlags.Kubeconfig)
err = deleteSecret(hostFactory, cluster.Spec.SecretRef.Name, unjoinFlags.HostSystemNamespace)
err = deleteSecret(hostFactory, cluster.Spec.SecretRef.Name, unjoinFlags.FederationSystemNamespace)
if isNotFound(err) {
fmt.Fprintf(cmdErr, "WARNING: secret %q not found in the host cluster, so it couldn't be deleted", cluster.Spec.SecretRef.Name)
} else if err != nil {

View File

@ -76,18 +76,18 @@ func (a *adminConfig) HostFactory(host, kubeconfigPath string) cmdutil.Factory {
// SubcommandFlags holds the flags required by the subcommands of
// `kubefed`.
type SubcommandFlags struct {
Name string
Host string
HostSystemNamespace string
Kubeconfig string
Name string
Host string
FederationSystemNamespace string
Kubeconfig string
}
// AddSubcommandFlags adds the definition for `kubefed` subcommand
// flags.
func AddSubcommandFlags(cmd *cobra.Command) {
cmd.Flags().String("kubeconfig", "", "Path to the kubeconfig file to use for CLI requests.")
cmd.Flags().String("host", "", "Host cluster context")
cmd.Flags().String("host-system-namespace", "federation-system", "Namespace in the host cluster where the federation system components are installed")
cmd.Flags().String("host-cluster-context", "", "Host cluster context")
cmd.Flags().String("federation-system-namespace", "federation-system", "Namespace in the host cluster where the federation system components are installed")
}
// GetSubcommandFlags retrieves the command line flag values for the
@ -98,10 +98,10 @@ func GetSubcommandFlags(cmd *cobra.Command, args []string) (*SubcommandFlags, er
return nil, err
}
return &SubcommandFlags{
Name: name,
Host: cmdutil.GetFlagString(cmd, "host"),
HostSystemNamespace: cmdutil.GetFlagString(cmd, "host-system-namespace"),
Kubeconfig: cmdutil.GetFlagString(cmd, "kubeconfig"),
Name: name,
Host: cmdutil.GetFlagString(cmd, "host-cluster-context"),
FederationSystemNamespace: cmdutil.GetFlagString(cmd, "federation-system-namespace"),
Kubeconfig: cmdutil.GetFlagString(cmd, "kubeconfig"),
}, nil
}

View File

@ -135,6 +135,7 @@ dns-bind-address
dns-port
dns-provider
dns-provider-config
dns-zone-name
docker-email
docker-endpoint
docker-exec-handler
@ -207,6 +208,7 @@ federated-api-burst
federated-api-qps
federated-kube-context
federation-name
federation-system-namespace
file-check-frequency
file-suffix
file_content_in_loop