diff --git a/federation/pkg/kubefed/init/init.go b/federation/pkg/kubefed/init/init.go index 14fb0066efd..60a738460dc 100644 --- a/federation/pkg/kubefed/init/init.go +++ b/federation/pkg/kubefed/init/init.go @@ -33,7 +33,9 @@ package init import ( "fmt" "io" + "io/ioutil" "net" + "os" "sort" "strconv" "strings" @@ -83,6 +85,8 @@ const ( apiserverServiceTypeFlag = "api-server-service-type" apiserverAdvertiseAddressFlag = "api-server-advertise-address" + + dnsProviderSecretName = "federation-dns-provider.conf" ) var ( @@ -129,6 +133,7 @@ type initFederationOptions struct { dnsZoneName string image string dnsProvider string + dnsProviderConfig string etcdPVCapacity string etcdPersistentStorage bool dryRun bool @@ -148,6 +153,7 @@ func (o *initFederationOptions) Bind(flags *pflag.FlagSet) { flags.StringVar(&o.dnsZoneName, "dns-zone-name", "", "DNS suffix for this federation. Federated Service DNS names are published with this suffix.") flags.StringVar(&o.image, "image", defaultImage, "Image to use for federation API server and controller manager binaries.") flags.StringVar(&o.dnsProvider, "dns-provider", "google-clouddns", "Dns provider to be used for this deployment.") + flags.StringVar(&o.dnsProviderConfig, "dns-provider-config", "", "Config file path on local file system for configuring DNS provider.") flags.StringVar(&o.etcdPVCapacity, "etcd-pv-capacity", "10Gi", "Size of persistent volume claim to be used for etcd.") flags.BoolVar(&o.etcdPersistentStorage, "etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.") flags.BoolVar(&o.dryRun, "dry-run", false, "dry run without sending commands to server.") @@ -211,11 +217,17 @@ func (i *initFederation) Complete(cmd *cobra.Command, args []string) error { i.options.apiServerOverrides, err = marshallOverrides(i.options.apiServerOverridesString) if err != nil { - return fmt.Errorf("Error marshalling --apiserver-arg-overrides: %v", err) + return fmt.Errorf("error marshalling --apiserver-arg-overrides: %v", err) } i.options.controllerManagerOverrides, err = marshallOverrides(i.options.controllerManagerOverridesString) if err != nil { - return fmt.Errorf("Error marshalling --controllermanager-arg-overrides: %v", err) + return fmt.Errorf("error marshalling --controllermanager-arg-overrides: %v", err) + } + + if i.options.dnsProviderConfig != "" { + if _, err := os.Stat(i.options.dnsProviderConfig); err != nil { + return fmt.Errorf("error reading file provided to --dns-provider-config flag, err: %v", err) + } } return nil @@ -236,6 +248,14 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error { cmName := fmt.Sprintf("%s-controller-manager", i.commonOptions.Name) cmKubeconfigName := fmt.Sprintf("%s-kubeconfig", cmName) + var dnsProviderConfigBytes []byte + if i.options.dnsProviderConfig != "" { + dnsProviderConfigBytes, err = ioutil.ReadFile(i.options.dnsProviderConfig) + if err != nil { + return fmt.Errorf("Error reading file provided to --dns-provider-config flag, err: %v", err) + } + } + // 1. Create a namespace for federation system components _, err = createNamespace(hostClientset, i.commonOptions.FederationSystemNamespace, i.options.dryRun) if err != nil { @@ -305,8 +325,14 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error { return err } - // 7c. Create federation controller manager deployment. - _, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, i.options.controllerManagerOverrides, i.options.dryRun) + // 7c. Create a dns-provider config secret + dnsProviderSecret, err := createDNSProviderConfigSecret(hostClientset, i.commonOptions.FederationSystemNamespace, dnsProviderSecretName, dnsProviderConfigBytes, i.options.dryRun) + if err != nil { + return err + } + + // 7d. Create federation controller manager deployment. + _, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun) if err != nil { return err } @@ -723,14 +749,13 @@ func createRoleBindings(clientset *client.Clientset, namespace, saName string, d return newRole, newRolebinding, err } -func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) { +func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) { command := []string{ "/hyperkube", "federation-controller-manager", } argsMap := map[string]string{ - "--kubeconfig": "/etc/federation/controller-manager/kubeconfig", - "--dns-provider-config": "", + "--kubeconfig": "/etc/federation/controller-manager/kubeconfig", } argsMap["--master"] = fmt.Sprintf("https://%s", svcName) @@ -798,6 +823,11 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa if dryRun { return dep, nil } + + if dnsProviderSecret != nil { + dep = addDNSProviderConfig(dep, dnsProviderSecret.Name) + } + return clientset.Extensions().Deployments(namespace).Create(dep) } @@ -935,3 +965,58 @@ func updateKubeconfig(config util.AdminConfig, name, endpoint, kubeConfigPath st return nil } + +func createDNSProviderConfigSecret(clientset *client.Clientset, namespace, name string, dnsProviderConfigBytes []byte, dryRun bool) (*api.Secret, error) { + if dnsProviderConfigBytes == nil { + return nil, nil + } + + secretSpec := &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + name: dnsProviderConfigBytes, + }, + } + + var secret *api.Secret + var err error + if !dryRun { + secret, err = clientset.Core().Secrets(namespace).Create(secretSpec) + if err != nil { + return nil, err + } + } + return secret, nil +} + +func addDNSProviderConfig(dep *extensions.Deployment, secretName string) *extensions.Deployment { + const ( + dnsProviderConfigVolume = "config-volume" + dnsProviderConfigMountPath = "/etc/federation/dns-provider" + ) + + // Create a volume from dns-provider secret + volume := api.Volume{ + Name: dnsProviderConfigVolume, + VolumeSource: api.VolumeSource{ + Secret: &api.SecretVolumeSource{ + SecretName: secretName, + }, + }, + } + dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, volume) + + // Mount dns-provider secret volume to controller-manager container + volumeMount := api.VolumeMount{ + Name: dnsProviderConfigVolume, + MountPath: dnsProviderConfigMountPath, + ReadOnly: true, + } + dep.Spec.Template.Spec.Containers[0].VolumeMounts = append(dep.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount) + dep.Spec.Template.Spec.Containers[0].Command = append(dep.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--dns-provider-config=%s/%s", dnsProviderConfigMountPath, secretName)) + + return dep +} diff --git a/federation/pkg/kubefed/init/init_test.go b/federation/pkg/kubefed/init/init_test.go index e67dc26c45d..bda1cc3715d 100644 --- a/federation/pkg/kubefed/init/init_test.go +++ b/federation/pkg/kubefed/init/init_test.go @@ -26,6 +26,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "sort" "strconv" "strings" @@ -90,6 +91,7 @@ func TestInitFederation(t *testing.T) { etcdPersistence string expectedErr string dnsProvider string + dnsProviderConfig string storageBackend string dryRun string apiserverArgOverrides string @@ -107,6 +109,7 @@ func TestInitFederation(t *testing.T) { etcdPersistence: "true", expectedErr: "", dnsProvider: "test-dns-provider", + dnsProviderConfig: "dns-provider.conf", storageBackend: "etcd2", dryRun: "", apiserverArgOverrides: "--client-ca-file=override,--log-dir=override", @@ -200,7 +203,15 @@ func TestInitFederation(t *testing.T) { } else { dnsProvider = "google-clouddns" //default value of dns-provider } - hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides) + if tc.dnsProviderConfig != "" { + tmpfile, err := ioutil.TempFile("", tc.dnsProviderConfig) + if err != nil { + t.Fatalf("[%d] unexpected error: %v", i, err) + } + tc.dnsProviderConfig = tmpfile.Name() + defer os.Remove(tmpfile.Name()) + } + hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides) if err != nil { t.Fatalf("[%d] unexpected error: %v", i, err) } @@ -225,6 +236,9 @@ func TestInitFederation(t *testing.T) { if tc.dnsProvider != "" { cmd.Flags().Set("dns-provider", tc.dnsProvider) } + if tc.dnsProviderConfig != "" { + cmd.Flags().Set("dns-provider-config", tc.dnsProviderConfig) + } if tc.etcdPVCapacity != "" { cmd.Flags().Set("etcd-pv-capacity", tc.etcdPVCapacity) } @@ -565,7 +579,7 @@ func TestCertsHTTPS(t *testing.T) { } } -func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) { +func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, dnsProviderConfig, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) { svcName := federationName + "-apiserver" svcUrlPrefix := "/api/v1/namespaces/federation-system/services" credSecretName := svcName + "-credentials" @@ -650,6 +664,18 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na Data: nil, } + cmDNSProviderSecret := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: testapi.Default.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: dnsProviderSecretName, + Namespace: namespaceName, + }, + Data: nil, + } + pvc := v1.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ Kind: "PersistentVolumeClaim", @@ -786,7 +812,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na sort.Strings(apiserverArgs) apiserverCommand = append(apiserverCommand, apiserverArgs...) - apiserver := v1beta1.Deployment{ + apiserver := &v1beta1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: testapi.Extensions.GroupVersion().String(), @@ -882,7 +908,6 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na cmArgs := []string{ "--kubeconfig=/etc/federation/controller-manager/kubeconfig", - "--dns-provider-config=", fmt.Sprintf("--federation-name=%s", federationName), fmt.Sprintf("--zone-name=%s", dnsZoneName), fmt.Sprintf("--master=https://%s", svcName), @@ -899,7 +924,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na cmCommand = append(cmCommand, cmArgs...) cmName := federationName + "-controller-manager" - cm := v1beta1.Deployment{ + cm := &v1beta1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: testapi.Extensions.GroupVersion().String(), @@ -958,6 +983,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na }, }, } + if dnsProviderConfig != "" { + cm = addDNSProviderConfigTest(cm, cmDNSProviderSecret.Name) + } podList := v1.PodList{} apiServerPod := v1.Pod{ @@ -1061,6 +1089,8 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na want = credSecret case cmKubeconfigSecretName: want = cmKubeconfigSecret + case dnsProviderSecretName: + want = cmDNSProviderSecret } if !apiequality.Semantic.DeepEqual(got, want) { return nil, fmt.Errorf("Unexpected secret object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, want)) @@ -1092,9 +1122,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na } switch got.Name { case svcName: - want = apiserver + want = *apiserver case cmName: - want = cm + want = *cm } if !apiequality.Semantic.DeepEqual(got, want) { return nil, fmt.Errorf("Unexpected deployment object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, want)) @@ -1354,3 +1384,33 @@ func getEndpoint(apiserverServiceType v1.ServiceType, lbIP, advertiseAddress str } return endpoint } + +// TODO: Reuse the function addDNSProviderConfig once that function is converted to use versioned objects. +func addDNSProviderConfigTest(dep *v1beta1.Deployment, secretName string) *v1beta1.Deployment { + const ( + dnsProviderConfigVolume = "config-volume" + dnsProviderConfigMountPath = "/etc/federation/dns-provider" + ) + + // Create a volume from dns-provider secret + volume := v1.Volume{ + Name: dnsProviderConfigVolume, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: secretName, + }, + }, + } + dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, volume) + + // Mount dns-provider secret volume to controller-manager container + volumeMount := v1.VolumeMount{ + Name: dnsProviderConfigVolume, + MountPath: dnsProviderConfigMountPath, + ReadOnly: true, + } + dep.Spec.Template.Spec.Containers[0].VolumeMounts = append(dep.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount) + dep.Spec.Template.Spec.Containers[0].Command = append(dep.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--dns-provider-config=%s/%s", dnsProviderConfigMountPath, secretName)) + + return dep +}