Merge pull request #40528 from shashidharatd/kubefed-2

Automatic merge from submit-queue (batch tested with PRs 41954, 40528, 41875, 41165, 41877) [Federation][kubefed] Support configuring dns-provider **What this PR does / why we need it**: Some environments might need to configure the dns-provider using custom configurations for deploying federation control plane. This PR will facilitate such scenarios. please refer to #40620 Now we can pass dns provider configuration using `dns-provider-config` flag to `kubefed init` **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #40620 **Release note**: ``` [Federation] Introduced a new flag --dns-provider-config to kubefed to configure dns provider via a config file in local file system. ``` cc @kubernetes/sig-federation-misc @madhusudancs @irfanurrehman @marun
2025-09-14 05:36:12 +00:00 · 2017-02-26 14:54:51 -08:00
parent 1fd3eb3451 d3adaeafe0
commit 267608029b
2 changed files with 159 additions and 14 deletions
--- a/federation/pkg/kubefed/init/init.go
+++ b/federation/pkg/kubefed/init/init.go
@@ -33,7 +33,9 @@ package init
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -83,6 +85,8 @@ const (

 	apiserverServiceTypeFlag      = "api-server-service-type"
 	apiserverAdvertiseAddressFlag = "api-server-advertise-address"
+
+	dnsProviderSecretName = "federation-dns-provider.conf"
 )

 var (
@@ -129,6 +133,7 @@ type initFederationOptions struct {
 	dnsZoneName                      string
 	image                            string
 	dnsProvider                      string
+	dnsProviderConfig                string
 	etcdPVCapacity                   string
 	etcdPersistentStorage            bool
 	dryRun                           bool
@@ -148,6 +153,7 @@ func (o *initFederationOptions) Bind(flags *pflag.FlagSet) {
 	flags.StringVar(&o.dnsZoneName, "dns-zone-name", "", "DNS suffix for this federation. Federated Service DNS names are published with this suffix.")
 	flags.StringVar(&o.image, "image", defaultImage, "Image to use for federation API server and controller manager binaries.")
 	flags.StringVar(&o.dnsProvider, "dns-provider", "google-clouddns", "Dns provider to be used for this deployment.")
+	flags.StringVar(&o.dnsProviderConfig, "dns-provider-config", "", "Config file path on local file system for configuring DNS provider.")
 	flags.StringVar(&o.etcdPVCapacity, "etcd-pv-capacity", "10Gi", "Size of persistent volume claim to be used for etcd.")
 	flags.BoolVar(&o.etcdPersistentStorage, "etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.")
 	flags.BoolVar(&o.dryRun, "dry-run", false, "dry run without sending commands to server.")
@@ -211,11 +217,17 @@ func (i *initFederation) Complete(cmd *cobra.Command, args []string) error {

 	i.options.apiServerOverrides, err = marshallOverrides(i.options.apiServerOverridesString)
 	if err != nil {
-		return fmt.Errorf("Error marshalling --apiserver-arg-overrides: %v", err)
+		return fmt.Errorf("error marshalling --apiserver-arg-overrides: %v", err)
 	}
 	i.options.controllerManagerOverrides, err = marshallOverrides(i.options.controllerManagerOverridesString)
 	if err != nil {
-		return fmt.Errorf("Error marshalling --controllermanager-arg-overrides: %v", err)
+		return fmt.Errorf("error marshalling --controllermanager-arg-overrides: %v", err)
+	}
+
+	if i.options.dnsProviderConfig != "" {
+		if _, err := os.Stat(i.options.dnsProviderConfig); err != nil {
+			return fmt.Errorf("error reading file provided to --dns-provider-config flag, err: %v", err)
+		}
 	}

 	return nil
@@ -236,6 +248,14 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {
 	cmName := fmt.Sprintf("%s-controller-manager", i.commonOptions.Name)
 	cmKubeconfigName := fmt.Sprintf("%s-kubeconfig", cmName)

+	var dnsProviderConfigBytes []byte
+	if i.options.dnsProviderConfig != "" {
+		dnsProviderConfigBytes, err = ioutil.ReadFile(i.options.dnsProviderConfig)
+		if err != nil {
+			return fmt.Errorf("Error reading file provided to --dns-provider-config flag, err: %v", err)
+		}
+	}
+
 	// 1. Create a namespace for federation system components
 	_, err = createNamespace(hostClientset, i.commonOptions.FederationSystemNamespace, i.options.dryRun)
 	if err != nil {
@@ -305,8 +325,14 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {
 		return err
 	}

-	// 7c. Create federation controller manager deployment.
-	_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, i.options.controllerManagerOverrides, i.options.dryRun)
+	// 7c. Create a dns-provider config secret
+	dnsProviderSecret, err := createDNSProviderConfigSecret(hostClientset, i.commonOptions.FederationSystemNamespace, dnsProviderSecretName, dnsProviderConfigBytes, i.options.dryRun)
+	if err != nil {
+		return err
+	}
+
+	// 7d. Create federation controller manager deployment.
+	_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
 	if err != nil {
 		return err
 	}
@@ -722,14 +748,13 @@ func createRoleBindings(clientset *client.Clientset, namespace, saName string, d
 	return newRole, newRolebinding, err
 }

-func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
+func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
 	command := []string{
 		"/hyperkube",
 		"federation-controller-manager",
 	}
 	argsMap := map[string]string{
-		"--kubeconfig":          "/etc/federation/controller-manager/kubeconfig",
-		"--dns-provider-config": "",
+		"--kubeconfig": "/etc/federation/controller-manager/kubeconfig",
 	}

 	argsMap["--master"] = fmt.Sprintf("https://%s", svcName)
@@ -797,6 +822,11 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa
 	if dryRun {
 		return dep, nil
 	}
+
+	if dnsProviderSecret != nil {
+		dep = addDNSProviderConfig(dep, dnsProviderSecret.Name)
+	}
+
 	return clientset.Extensions().Deployments(namespace).Create(dep)
 }

@@ -934,3 +964,58 @@ func updateKubeconfig(config util.AdminConfig, name, endpoint, kubeConfigPath st

 	return nil
 }
+
+func createDNSProviderConfigSecret(clientset *client.Clientset, namespace, name string, dnsProviderConfigBytes []byte, dryRun bool) (*api.Secret, error) {
+	if dnsProviderConfigBytes == nil {
+		return nil, nil
+	}
+
+	secretSpec := &api.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			name: dnsProviderConfigBytes,
+		},
+	}
+
+	var secret *api.Secret
+	var err error
+	if !dryRun {
+		secret, err = clientset.Core().Secrets(namespace).Create(secretSpec)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return secret, nil
+}
+
+func addDNSProviderConfig(dep *extensions.Deployment, secretName string) *extensions.Deployment {
+	const (
+		dnsProviderConfigVolume    = "config-volume"
+		dnsProviderConfigMountPath = "/etc/federation/dns-provider"
+	)
+
+	// Create a volume from dns-provider secret
+	volume := api.Volume{
+		Name: dnsProviderConfigVolume,
+		VolumeSource: api.VolumeSource{
+			Secret: &api.SecretVolumeSource{
+				SecretName: secretName,
+			},
+		},
+	}
+	dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, volume)
+
+	// Mount dns-provider secret volume to controller-manager container
+	volumeMount := api.VolumeMount{
+		Name:      dnsProviderConfigVolume,
+		MountPath: dnsProviderConfigMountPath,
+		ReadOnly:  true,
+	}
+	dep.Spec.Template.Spec.Containers[0].VolumeMounts = append(dep.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount)
+	dep.Spec.Template.Spec.Containers[0].Command = append(dep.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--dns-provider-config=%s/%s", dnsProviderConfigMountPath, secretName))
+
+	return dep
+}
--- a/federation/pkg/kubefed/init/init_test.go
+++ b/federation/pkg/kubefed/init/init_test.go
@@ -26,6 +26,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -90,6 +91,7 @@ func TestInitFederation(t *testing.T) {
 		etcdPersistence       string
 		expectedErr           string
 		dnsProvider           string
+		dnsProviderConfig     string
 		storageBackend        string
 		dryRun                string
 		apiserverArgOverrides string
@@ -107,6 +109,7 @@ func TestInitFederation(t *testing.T) {
 			etcdPersistence:       "true",
 			expectedErr:           "",
 			dnsProvider:           "test-dns-provider",
+			dnsProviderConfig:     "dns-provider.conf",
 			storageBackend:        "etcd2",
 			dryRun:                "",
 			apiserverArgOverrides: "--client-ca-file=override,--log-dir=override",
@@ -200,7 +203,15 @@ func TestInitFederation(t *testing.T) {
 		} else {
 			dnsProvider = "google-clouddns" //default value of dns-provider
 		}
-		hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides)
+		if tc.dnsProviderConfig != "" {
+			tmpfile, err := ioutil.TempFile("", tc.dnsProviderConfig)
+			if err != nil {
+				t.Fatalf("[%d] unexpected error: %v", i, err)
+			}
+			tc.dnsProviderConfig = tmpfile.Name()
+			defer os.Remove(tmpfile.Name())
+		}
+		hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -225,6 +236,9 @@ func TestInitFederation(t *testing.T) {
 		if tc.dnsProvider != "" {
 			cmd.Flags().Set("dns-provider", tc.dnsProvider)
 		}
+		if tc.dnsProviderConfig != "" {
+			cmd.Flags().Set("dns-provider-config", tc.dnsProviderConfig)
+		}
 		if tc.etcdPVCapacity != "" {
 			cmd.Flags().Set("etcd-pv-capacity", tc.etcdPVCapacity)
 		}
@@ -565,7 +579,7 @@ func TestCertsHTTPS(t *testing.T) {
 	}
 }

-func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) {
+func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, dnsProviderConfig, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) {
 	svcName := federationName + "-apiserver"
 	svcUrlPrefix := "/api/v1/namespaces/federation-system/services"
 	credSecretName := svcName + "-credentials"
@@ -650,6 +664,18 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 		Data: nil,
 	}

+	cmDNSProviderSecret := v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: testapi.Default.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      dnsProviderSecretName,
+			Namespace: namespaceName,
+		},
+		Data: nil,
+	}
+
 	pvc := v1.PersistentVolumeClaim{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "PersistentVolumeClaim",
@@ -785,7 +811,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 	sort.Strings(apiserverArgs)
 	apiserverCommand = append(apiserverCommand, apiserverArgs...)

-	apiserver := v1beta1.Deployment{
+	apiserver := &v1beta1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: testapi.Extensions.GroupVersion().String(),
@@ -881,7 +907,6 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na

 	cmArgs := []string{
 		"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
-		"--dns-provider-config=",
 		fmt.Sprintf("--federation-name=%s", federationName),
 		fmt.Sprintf("--zone-name=%s", dnsZoneName),
 		fmt.Sprintf("--master=https://%s", svcName),
@@ -898,7 +923,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 	cmCommand = append(cmCommand, cmArgs...)

 	cmName := federationName + "-controller-manager"
-	cm := v1beta1.Deployment{
+	cm := &v1beta1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: testapi.Extensions.GroupVersion().String(),
@@ -957,6 +982,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 			},
 		},
 	}
+	if dnsProviderConfig != "" {
+		cm = addDNSProviderConfigTest(cm, cmDNSProviderSecret.Name)
+	}

 	podList := v1.PodList{}
 	apiServerPod := v1.Pod{
@@ -1060,6 +1088,8 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 					want = credSecret
 				case cmKubeconfigSecretName:
 					want = cmKubeconfigSecret
+				case dnsProviderSecretName:
+					want = cmDNSProviderSecret
 				}
 				if !apiequality.Semantic.DeepEqual(got, want) {
 					return nil, fmt.Errorf("Unexpected secret object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, want))
@@ -1091,9 +1121,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 				}
 				switch got.Name {
 				case svcName:
-					want = apiserver
+					want = *apiserver
 				case cmName:
-					want = cm
+					want = *cm
 				}
 				if !apiequality.Semantic.DeepEqual(got, want) {
 					return nil, fmt.Errorf("Unexpected deployment object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, want))
@@ -1353,3 +1383,33 @@ func getEndpoint(apiserverServiceType v1.ServiceType, lbIP, advertiseAddress str
 	}
 	return endpoint
 }
+
+// TODO: Reuse the function addDNSProviderConfig once that function is converted to use versioned objects.
+func addDNSProviderConfigTest(dep *v1beta1.Deployment, secretName string) *v1beta1.Deployment {
+	const (
+		dnsProviderConfigVolume    = "config-volume"
+		dnsProviderConfigMountPath = "/etc/federation/dns-provider"
+	)
+
+	// Create a volume from dns-provider secret
+	volume := v1.Volume{
+		Name: dnsProviderConfigVolume,
+		VolumeSource: v1.VolumeSource{
+			Secret: &v1.SecretVolumeSource{
+				SecretName: secretName,
+			},
+		},
+	}
+	dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, volume)
+
+	// Mount dns-provider secret volume to controller-manager container
+	volumeMount := v1.VolumeMount{
+		Name:      dnsProviderConfigVolume,
+		MountPath: dnsProviderConfigMountPath,
+		ReadOnly:  true,
+	}
+	dep.Spec.Template.Spec.Containers[0].VolumeMounts = append(dep.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount)
+	dep.Spec.Template.Spec.Containers[0].Command = append(dep.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--dns-provider-config=%s/%s", dnsProviderConfigMountPath, secretName))
+
+	return dep
+}