diff --git a/federation/pkg/kubefed/init/BUILD b/federation/pkg/kubefed/init/BUILD index d216563f3d1..7c844d37147 100644 --- a/federation/pkg/kubefed/init/BUILD +++ b/federation/pkg/kubefed/init/BUILD @@ -54,6 +54,7 @@ go_test( "//vendor:k8s.io/apimachinery/pkg/api/resource", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/util/diff", + "//vendor:k8s.io/apimachinery/pkg/util/sets", "//vendor:k8s.io/client-go/dynamic", "//vendor:k8s.io/client-go/rest/fake", "//vendor:k8s.io/client-go/tools/clientcmd", diff --git a/federation/pkg/kubefed/init/init.go b/federation/pkg/kubefed/init/init.go index f0d0eec7913..6307c752b86 100644 --- a/federation/pkg/kubefed/init/init.go +++ b/federation/pkg/kubefed/init/init.go @@ -57,6 +57,7 @@ import ( "k8s.io/kubernetes/pkg/version" "github.com/spf13/cobra" + "sort" ) const ( @@ -141,6 +142,8 @@ func NewCmdInit(cmdOut io.Writer, config util.AdminConfig) *cobra.Command { cmd.Flags().String("etcd-pv-capacity", "10Gi", "Size of persistent volume claim to be used for etcd.") cmd.Flags().Bool("etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.") cmd.Flags().Bool("dry-run", false, "dry run without sending commands to server.") + cmd.Flags().String("apiserver-arg-overrides", "", "comma separated list of federation-apiserver arguments to override: Example \"--arg1=value1,--arg2=value2...\"") + cmd.Flags().String("controllermanager-arg-overrides", "", "comma separated list of federation-controller-manager arguments to override: Example \"--arg1=value1,--arg2=value2...\"") cmd.Flags().String("storage-backend", "etcd2", "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.") cmd.Flags().String(apiserverServiceTypeFlag, string(v1.ServiceTypeLoadBalancer), "The type of service to create for federation API server. Options: 'LoadBalancer' (default), 'NodePort'.") cmd.Flags().String(apiserverAdvertiseAddressFlag, "", "Preferred address to advertise api server nodeport service. Valid only if '"+apiserverServiceTypeFlag+"=NodePort'.") @@ -184,6 +187,14 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman return fmt.Errorf("%s should be passed only with '%s=NodePort'", apiserverAdvertiseAddressFlag, apiserverServiceTypeFlag) } } + apiserverArgOverrides, err := marshallOverrides(cmdutil.GetFlagString(cmd, "apiserver-arg-overrides")) + if err != nil { + return fmt.Errorf("Error marshalling --apiserver-arg-overrides: %v", err) + } + cmArgOverrides, err := marshallOverrides(cmdutil.GetFlagString(cmd, "controllermanager-arg-overrides")) + if err != nil { + return fmt.Errorf("Error marshalling --controllermanager-arg-overrides: %v", err) + } hostFactory := config.HostFactory(initFlags.Host, initFlags.Kubeconfig) hostClientset, err := hostFactory.ClientSet() @@ -245,7 +256,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman } // 6. Create federation API server - _, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, pvc, dryRun) + _, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, apiserverArgOverrides, pvc, dryRun) if err != nil { return err } @@ -266,7 +277,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman } // 7c. Create federation controller manager deployment. - _, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, sa.Name, dryRun) + _, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, sa.Name, cmArgOverrides, dryRun) if err != nil { return err } @@ -518,24 +529,29 @@ func createPVC(clientset *client.Clientset, namespace, svcName, etcdPVCapacity s return clientset.Core().PersistentVolumeClaims(namespace).Create(pvc) } -func createAPIServer(clientset *client.Clientset, namespace, name, image, credentialsName, advertiseAddress, storageBackend string, pvc *api.PersistentVolumeClaim, dryRun bool) (*extensions.Deployment, error) { +func createAPIServer(clientset *client.Clientset, namespace, name, image, credentialsName, advertiseAddress, storageBackend string, argOverrides map[string]string, pvc *api.PersistentVolumeClaim, dryRun bool) (*extensions.Deployment, error) { command := []string{ "/hyperkube", "federation-apiserver", - "--bind-address=0.0.0.0", - "--etcd-servers=http://localhost:2379", - "--secure-port=443", - "--client-ca-file=/etc/federation/apiserver/ca.crt", - "--tls-cert-file=/etc/federation/apiserver/server.crt", - "--tls-private-key-file=/etc/federation/apiserver/server.key", - "--admission-control=NamespaceLifecycle", - fmt.Sprintf("--storage-backend=%s", storageBackend), + } + argsMap := map[string]string{ + "--bind-address": "0.0.0.0", + "--etcd-servers": "http://localhost:2379", + "--secure-port": "443", + "--client-ca-file": "/etc/federation/apiserver/ca.crt", + "--tls-cert-file": "/etc/federation/apiserver/server.crt", + "--tls-private-key-file": "/etc/federation/apiserver/server.key", + "--admission-control": "NamespaceLifecycle", } + argsMap["--storage-backend"] = storageBackend if advertiseAddress != "" { - command = append(command, fmt.Sprintf("--advertise-address=%s", advertiseAddress)) + argsMap["--advertise-address"] = advertiseAddress } + args := argMapsToArgStrings(argsMap, argOverrides) + command = append(command, args...) + dep := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -676,7 +692,24 @@ func createRoleBindings(clientset *client.Clientset, namespace, saName string, d return newRole, newRolebinding, err } -func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dryRun bool) (*extensions.Deployment, error) { +func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) { + command := []string{ + "/hyperkube", + "federation-controller-manager", + } + argsMap := map[string]string{ + "--kubeconfig": "/etc/federation/controller-manager/kubeconfig", + "--dns-provider-config": "", + } + + argsMap["--master"] = fmt.Sprintf("https://%s", svcName) + argsMap["--dns-provider"] = dnsProvider + argsMap["--federation-name"] = name + argsMap["--zone-name"] = dnsZoneName + + args := argMapsToArgStrings(argsMap, argOverrides) + command = append(command, args...) + dep := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: cmName, @@ -693,18 +726,9 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa Spec: api.PodSpec{ Containers: []api.Container{ { - Name: "controller-manager", - Image: image, - Command: []string{ - "/hyperkube", - "federation-controller-manager", - fmt.Sprintf("--master=https://%s", svcName), - "--kubeconfig=/etc/federation/controller-manager/kubeconfig", - fmt.Sprintf("--dns-provider=%s", dnsProvider), - "--dns-provider-config=", - fmt.Sprintf("--federation-name=%s", name), - fmt.Sprintf("--zone-name=%s", dnsZoneName), - }, + Name: "controller-manager", + Image: image, + Command: command, VolumeMounts: []api.VolumeMount{ { Name: kubeconfigName, @@ -746,6 +770,41 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa return clientset.Extensions().Deployments(namespace).Create(dep) } +func marshallOverrides(overrideArgString string) (map[string]string, error) { + if overrideArgString == "" { + return nil, nil + } + + argsMap := make(map[string]string) + overrideArgs := strings.Split(overrideArgString, ",") + for _, overrideArg := range overrideArgs { + splitArg := strings.Split(overrideArg, "=") + if len(splitArg) != 2 { + return nil, fmt.Errorf("wrong format for override arg: %s", overrideArg) + } + key := strings.TrimSpace(splitArg[0]) + val := strings.TrimSpace(splitArg[1]) + if len(key) == 0 { + return nil, fmt.Errorf("wrong format for override arg: %s, arg name cannot be empty", overrideArg) + } + argsMap[key] = val + } + return argsMap, nil +} + +func argMapsToArgStrings(argsMap, overrides map[string]string) []string { + for key, val := range overrides { + argsMap[key] = val + } + args := []string{} + for key, value := range argsMap { + args = append(args, fmt.Sprintf("%s=%s", key, value)) + } + // This is needed for the unit test deep copy to get an exact match + sort.Strings(args) + return args +} + func waitForPods(clientset *client.Clientset, fedPods []string, namespace string) error { err := wait.PollInfinite(podWaitInterval, func() (bool, error) { podCheck := len(fedPods) diff --git a/federation/pkg/kubefed/init/init_test.go b/federation/pkg/kubefed/init/init_test.go index 46dd1d20d20..85e5eec273f 100644 --- a/federation/pkg/kubefed/init/init_test.go +++ b/federation/pkg/kubefed/init/init_test.go @@ -26,6 +26,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "sort" "strconv" "strings" "testing" @@ -36,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest/fake" "k8s.io/client-go/tools/clientcmd" @@ -76,35 +78,39 @@ func TestInitFederation(t *testing.T) { defer kubefedtesting.RemoveFakeKubeconfigFiles(fakeKubeFiles) testCases := []struct { - federation string - kubeconfigGlobal string - kubeconfigExplicit string - dnsZoneName string - lbIP string - apiserverServiceType v1.ServiceType - advertiseAddress string - image string - etcdPVCapacity string - etcdPersistence string - expectedErr string - dnsProvider string - storageBackend string - dryRun string + federation string + kubeconfigGlobal string + kubeconfigExplicit string + dnsZoneName string + lbIP string + apiserverServiceType v1.ServiceType + advertiseAddress string + image string + etcdPVCapacity string + etcdPersistence string + expectedErr string + dnsProvider string + storageBackend string + dryRun string + apiserverArgOverrides string + cmArgOverrides string }{ { - federation: "union", - kubeconfigGlobal: fakeKubeFiles[0], - kubeconfigExplicit: "", - dnsZoneName: "example.test.", - lbIP: lbIP, - apiserverServiceType: v1.ServiceTypeLoadBalancer, - image: "example.test/foo:bar", - etcdPVCapacity: "5Gi", - etcdPersistence: "true", - expectedErr: "", - dnsProvider: "test-dns-provider", - storageBackend: "etcd2", - dryRun: "", + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + lbIP: lbIP, + apiserverServiceType: v1.ServiceTypeLoadBalancer, + image: "example.test/foo:bar", + etcdPVCapacity: "5Gi", + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd2", + dryRun: "", + apiserverArgOverrides: "--client-ca-file=override,--log-dir=override", + cmArgOverrides: "--dns-provider=override,--log-dir=override", }, { federation: "union", @@ -194,7 +200,7 @@ func TestInitFederation(t *testing.T) { } else { dnsProvider = "google-clouddns" //default value of dns-provider } - hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend) + hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides) if err != nil { t.Fatalf("[%d] unexpected error: %v", i, err) } @@ -210,6 +216,9 @@ func TestInitFederation(t *testing.T) { cmd.Flags().Set("host-cluster-context", "substrate") cmd.Flags().Set("dns-zone-name", tc.dnsZoneName) cmd.Flags().Set("image", tc.image) + cmd.Flags().Set("apiserver-arg-overrides", tc.apiserverArgOverrides) + cmd.Flags().Set("controllermanager-arg-overrides", tc.cmArgOverrides) + if tc.storageBackend != "" { cmd.Flags().Set("storage-backend", tc.storageBackend) } @@ -259,6 +268,64 @@ func TestInitFederation(t *testing.T) { } } +func TestMarshallAndMergeOverrides(t *testing.T) { + testCases := []struct { + overrideParams string + expectedSet sets.String + expectedErr string + }{ + { + overrideParams: "valid-format-param1=override1,valid-format-param2=override2", + expectedSet: sets.NewString("arg2=val2", "arg1=val1", "valid-format-param1=override1", "valid-format-param2=override2"), + expectedErr: "", + }, + { + overrideParams: "valid-format-param1=override1,arg1=override1", + expectedSet: sets.NewString("arg2=val2", "arg1=override1", "valid-format-param1=override1"), + expectedErr: "", + }, + { + overrideParams: "zero-value-arg=", + expectedSet: sets.NewString("arg2=val2", "arg1=val1", "zero-value-arg="), + expectedErr: "", + }, + { + overrideParams: "wrong-format-arg", + expectedErr: "wrong format for override arg: wrong-format-arg", + }, + { + overrideParams: "wrong-format-arg=override=wrong-format-arg=override", + expectedErr: "wrong format for override arg: wrong-format-arg=override=wrong-format-arg=override", + }, + { + overrideParams: "=wrong-format-only-value", + expectedErr: "wrong format for override arg: =wrong-format-only-value, arg name cannot be empty", + }, + } + + for i, tc := range testCases { + args, err := marshallOverrides(tc.overrideParams) + if tc.expectedErr == "" { + origArgs := map[string]string{ + "arg1": "val1", + "arg2": "val2", + } + merged := argMapsToArgStrings(origArgs, args) + + got := sets.NewString(merged...) + want := tc.expectedSet + + if !got.Equal(want) { + t.Errorf("[%d] unexpected output: got: %v, want: %v", i, got, want) + } + } else { + if err.Error() != tc.expectedErr { + t.Errorf("[%d] unexpected error output: got: %s, want: %s", i, err.Error(), tc.expectedErr) + } + } + } +} + // TestCertsTLS tests TLS handshake with client authentication for any server // name. There is a separate test below to test the certificate generation // end-to-end over HTTPS. @@ -498,7 +565,7 @@ func TestCertsHTTPS(t *testing.T) { } } -func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) { +func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) { svcName := federationName + "-apiserver" svcUrlPrefix := "/api/v1/namespaces/federation-system/services" credSecretName := svcName + "-credentials" @@ -684,6 +751,40 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na nodeList := v1.NodeList{} nodeList.Items = append(nodeList.Items, node) + address := lbIp + if apiserverServiceType == v1.ServiceTypeNodePort { + if advertiseAddress != "" { + address = advertiseAddress + } else { + address = nodeIP + } + } + + apiserverCommand := []string{ + "/hyperkube", + "federation-apiserver", + } + apiserverArgs := []string{ + "--bind-address=0.0.0.0", + "--etcd-servers=http://localhost:2379", + "--secure-port=443", + "--tls-cert-file=/etc/federation/apiserver/server.crt", + "--tls-private-key-file=/etc/federation/apiserver/server.key", + "--admission-control=NamespaceLifecycle", + fmt.Sprintf("--storage-backend=%s", storageProvider), + fmt.Sprintf("--advertise-address=%s", address), + } + + if apiserverOverrideArg != "" { + apiserverArgs = append(apiserverArgs, "--client-ca-file=override") + apiserverArgs = append(apiserverArgs, "--log-dir=override") + + } else { + apiserverArgs = append(apiserverArgs, "--client-ca-file=/etc/federation/apiserver/ca.crt") + } + sort.Strings(apiserverArgs) + apiserverCommand = append(apiserverCommand, apiserverArgs...) + apiserver := v1beta1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -705,20 +806,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "apiserver", - Image: image, - Command: []string{ - "/hyperkube", - "federation-apiserver", - "--bind-address=0.0.0.0", - "--etcd-servers=http://localhost:2379", - "--secure-port=443", - "--client-ca-file=/etc/federation/apiserver/ca.crt", - "--tls-cert-file=/etc/federation/apiserver/server.crt", - "--tls-private-key-file=/etc/federation/apiserver/server.key", - "--admission-control=NamespaceLifecycle", - fmt.Sprintf("--storage-backend=%s", storageProvider), - }, + Name: "apiserver", + Image: image, + Command: apiserverCommand, Ports: []v1.ContainerPort{ { Name: "https", @@ -784,15 +874,28 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na } } - address := lbIp - if apiserverServiceType == v1.ServiceTypeNodePort { - if advertiseAddress != "" { - address = advertiseAddress - } else { - address = nodeIP - } + cmCommand := []string{ + "/hyperkube", + "federation-controller-manager", } - apiserver.Spec.Template.Spec.Containers[0].Command = append(apiserver.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--advertise-address=%s", address)) + + cmArgs := []string{ + "--kubeconfig=/etc/federation/controller-manager/kubeconfig", + "--dns-provider-config=", + fmt.Sprintf("--federation-name=%s", federationName), + fmt.Sprintf("--zone-name=%s", dnsZoneName), + fmt.Sprintf("--master=https://%s", svcName), + } + + if cmOverrideArg != "" { + cmArgs = append(cmArgs, "--dns-provider=override") + cmArgs = append(cmArgs, "--log-dir=override") + } else { + cmArgs = append(cmArgs, fmt.Sprintf("--dns-provider=%s", dnsProvider)) + } + + sort.Strings(cmArgs) + cmCommand = append(cmCommand, cmArgs...) cmName := federationName + "-controller-manager" cm := v1beta1.Deployment{ @@ -816,18 +919,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "controller-manager", - Image: image, - Command: []string{ - "/hyperkube", - "federation-controller-manager", - "--master=https://" + svcName, - "--kubeconfig=/etc/federation/controller-manager/kubeconfig", - fmt.Sprintf("--dns-provider=%s", dnsProvider), - "--dns-provider-config=", - fmt.Sprintf("--federation-name=%s", federationName), - fmt.Sprintf("--zone-name=%s", dnsZoneName), - }, + Name: "controller-manager", + Image: image, + Command: cmCommand, VolumeMounts: []v1.VolumeMount{ { Name: cmKubeconfigSecretName, diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1e03f0b38e8..3cc162de38e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -23,6 +23,7 @@ api-server-advertise-address api-server-service-type api-token api-version +apiserver-arg-overrides apiserver-count apiserver-count audit-log-maxage @@ -114,6 +115,7 @@ container-runtime container-runtime-endpoint contain-pod-resources contention-profiling +controllermanager-arg-overrides controller-start-interval cors-allowed-origins cpu-cfs-quota