github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 21:17:23 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #40917 from irfanurrehman/fed-init-flags-1

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 40917, 41181, 41123, 36592, 41183)

[Federation] Add override flags options to kubefed init

**What this PR does / why we need it**:
Allows modification of startup flags (of apiserver and controller manager) through kubefed 
**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #
https://github.com/kubernetes/kubernetes/issues/40398
**Special notes for your reviewer**:
I haven't removed the existing redundant flags now (for example --dns-zone-name) intentionally to avoid breaking any existing tests that might use them.
I guess that would be better done as a follow up PR.
@madhusudancs @marun @nikhiljindal 
**Release note**:

```
It is now possible for the user to modify any startup flag of federation-apiserver and federation-controller-manager when deployed through kubefed.
There are two new options introduced in kubefed:
--apiserver-arg-overrides and --controllermanager-arg-overrides
Any number of actual federation-apiserver or federation-controller-manager flags can be specified using these options.
Example:
kubefed init "-other options-" ----apiserver-arg-overrides "--flag1=value1,--flag2=value2"
```
This commit is contained in:
Kubernetes Submit Queue 2017-02-09 23:10:41 -08:00 committed by GitHub
commit c429a074e8
4 changed files with 244 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
federation/pkg/kubefed/init/BUILD
View File

@ -54,6 +54,7 @@ go_test(
"//vendor:k8s.io/apimachinery/pkg/api/resource",
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
"//vendor:k8s.io/apimachinery/pkg/util/diff",
"//vendor:k8s.io/apimachinery/pkg/util/sets",
"//vendor:k8s.io/client-go/dynamic",
"//vendor:k8s.io/client-go/rest/fake",
"//vendor:k8s.io/client-go/tools/clientcmd",

105
federation/pkg/kubefed/init/init.go
View File

@ -57,6 +57,7 @@ import (
"k8s.io/kubernetes/pkg/version"
"github.com/spf13/cobra"
"sort"
)
const (
@ -141,6 +142,8 @@ func NewCmdInit(cmdOut io.Writer, config util.AdminConfig) *cobra.Command {
cmd.Flags().String("etcd-pv-capacity", "10Gi", "Size of persistent volume claim to be used for etcd.")
cmd.Flags().Bool("etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.")
cmd.Flags().Bool("dry-run", false, "dry run without sending commands to server.")
cmd.Flags().String("apiserver-arg-overrides", "", "comma separated list of federation-apiserver arguments to override: Example \"--arg1=value1,--arg2=value2...\"")
cmd.Flags().String("controllermanager-arg-overrides", "", "comma separated list of federation-controller-manager arguments to override: Example \"--arg1=value1,--arg2=value2...\"")
cmd.Flags().String("storage-backend", "etcd2", "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.")
cmd.Flags().String(apiserverServiceTypeFlag, string(v1.ServiceTypeLoadBalancer), "The type of service to create for federation API server. Options: 'LoadBalancer' (default), 'NodePort'.")
cmd.Flags().String(apiserverAdvertiseAddressFlag, "", "Preferred address to advertise api server nodeport service. Valid only if '"+apiserverServiceTypeFlag+"=NodePort'.")
@ -184,6 +187,14 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
return fmt.Errorf("%s should be passed only with '%s=NodePort'", apiserverAdvertiseAddressFlag, apiserverServiceTypeFlag)
}
}
apiserverArgOverrides, err := marshallOverrides(cmdutil.GetFlagString(cmd, "apiserver-arg-overrides"))
if err != nil {
return fmt.Errorf("Error marshalling --apiserver-arg-overrides: %v", err)
}
cmArgOverrides, err := marshallOverrides(cmdutil.GetFlagString(cmd, "controllermanager-arg-overrides"))
if err != nil {
return fmt.Errorf("Error marshalling --controllermanager-arg-overrides: %v", err)
}
hostFactory := config.HostFactory(initFlags.Host, initFlags.Kubeconfig)
hostClientset, err := hostFactory.ClientSet()
@ -245,7 +256,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
}
// 6. Create federation API server
_, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, pvc, dryRun)
_, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, apiserverArgOverrides, pvc, dryRun)
if err != nil {
return err
}
@ -266,7 +277,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
}
// 7c. Create federation controller manager deployment.
_, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, sa.Name, dryRun)
_, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, sa.Name, cmArgOverrides, dryRun)
if err != nil {
return err
}
@ -518,24 +529,29 @@ func createPVC(clientset *client.Clientset, namespace, svcName, etcdPVCapacity s
return clientset.Core().PersistentVolumeClaims(namespace).Create(pvc)
}
func createAPIServer(clientset *client.Clientset, namespace, name, image, credentialsName, advertiseAddress, storageBackend string, pvc *api.PersistentVolumeClaim, dryRun bool) (*extensions.Deployment, error) {
func createAPIServer(clientset *client.Clientset, namespace, name, image, credentialsName, advertiseAddress, storageBackend string, argOverrides map[string]string, pvc *api.PersistentVolumeClaim, dryRun bool) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-apiserver",
"--bind-address=0.0.0.0",
"--etcd-servers=http://localhost:2379",
"--secure-port=443",
"--client-ca-file=/etc/federation/apiserver/ca.crt",
"--tls-cert-file=/etc/federation/apiserver/server.crt",
"--tls-private-key-file=/etc/federation/apiserver/server.key",
"--admission-control=NamespaceLifecycle",
fmt.Sprintf("--storage-backend=%s", storageBackend),
}
argsMap := map[string]string{
"--bind-address": "0.0.0.0",
"--etcd-servers": "http://localhost:2379",
"--secure-port": "443",
"--client-ca-file": "/etc/federation/apiserver/ca.crt",
"--tls-cert-file": "/etc/federation/apiserver/server.crt",
"--tls-private-key-file": "/etc/federation/apiserver/server.key",
"--admission-control": "NamespaceLifecycle",
}
argsMap["--storage-backend"] = storageBackend
if advertiseAddress != "" {
command = append(command, fmt.Sprintf("--advertise-address=%s", advertiseAddress))
argsMap["--advertise-address"] = advertiseAddress
}
args := argMapsToArgStrings(argsMap, argOverrides)
command = append(command, args...)
dep := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
@ -676,7 +692,24 @@ func createRoleBindings(clientset *client.Clientset, namespace, saName string, d
return newRole, newRolebinding, err
}
func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dryRun bool) (*extensions.Deployment, error) {
func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-controller-manager",
}
argsMap := map[string]string{
"--kubeconfig": "/etc/federation/controller-manager/kubeconfig",
"--dns-provider-config": "",
}
argsMap["--master"] = fmt.Sprintf("https://%s", svcName)
argsMap["--dns-provider"] = dnsProvider
argsMap["--federation-name"] = name
argsMap["--zone-name"] = dnsZoneName
args := argMapsToArgStrings(argsMap, argOverrides)
command = append(command, args...)
dep := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
@ -695,16 +728,7 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa
{
Name: "controller-manager",
Image: image,
Command: []string{
"/hyperkube",
"federation-controller-manager",
fmt.Sprintf("--master=https://%s", svcName),
"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
fmt.Sprintf("--dns-provider=%s", dnsProvider),
"--dns-provider-config=",
fmt.Sprintf("--federation-name=%s", name),
fmt.Sprintf("--zone-name=%s", dnsZoneName),
},
Command: command,
VolumeMounts: []api.VolumeMount{
{
Name: kubeconfigName,
@ -746,6 +770,41 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa
return clientset.Extensions().Deployments(namespace).Create(dep)
}
func marshallOverrides(overrideArgString string) (map[string]string, error) {
if overrideArgString == "" {
return nil, nil
}
argsMap := make(map[string]string)
overrideArgs := strings.Split(overrideArgString, ",")
for _, overrideArg := range overrideArgs {
splitArg := strings.Split(overrideArg, "=")
if len(splitArg) != 2 {
return nil, fmt.Errorf("wrong format for override arg: %s", overrideArg)
}
key := strings.TrimSpace(splitArg[0])
val := strings.TrimSpace(splitArg[1])
if len(key) == 0 {
return nil, fmt.Errorf("wrong format for override arg: %s, arg name cannot be empty", overrideArg)
}
argsMap[key] = val
}
return argsMap, nil
}
func argMapsToArgStrings(argsMap, overrides map[string]string) []string {
for key, val := range overrides {
argsMap[key] = val
}
args := []string{}
for key, value := range argsMap {
args = append(args, fmt.Sprintf("%s=%s", key, value))
}
// This is needed for the unit test deep copy to get an exact match
sort.Strings(args)
return args
}
func waitForPods(clientset *client.Clientset, fedPods []string, namespace string) error {
err := wait.PollInfinite(podWaitInterval, func() (bool, error) {
podCheck := len(fedPods)

156
federation/pkg/kubefed/init/init_test.go
View File

@ -26,6 +26,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"sort"
"strconv"
"strings"
"testing"
@ -36,6 +37,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest/fake"
"k8s.io/client-go/tools/clientcmd"
@ -90,6 +92,8 @@ func TestInitFederation(t *testing.T) {
dnsProvider string
storageBackend string
dryRun string
apiserverArgOverrides string
cmArgOverrides string
}{
{
federation: "union",
@ -105,6 +109,8 @@ func TestInitFederation(t *testing.T) {
dnsProvider: "test-dns-provider",
storageBackend: "etcd2",
dryRun: "",
apiserverArgOverrides: "--client-ca-file=override,--log-dir=override",
cmArgOverrides: "--dns-provider=override,--log-dir=override",
},
{
federation: "union",
@ -194,7 +200,7 @@ func TestInitFederation(t *testing.T) {
} else {
dnsProvider = "google-clouddns" //default value of dns-provider
}
hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend)
hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend, tc.apiserverArgOverrides, tc.cmArgOverrides)
if err != nil {
t.Fatalf("[%d] unexpected error: %v", i, err)
}
@ -210,6 +216,9 @@ func TestInitFederation(t *testing.T) {
cmd.Flags().Set("host-cluster-context", "substrate")
cmd.Flags().Set("dns-zone-name", tc.dnsZoneName)
cmd.Flags().Set("image", tc.image)
cmd.Flags().Set("apiserver-arg-overrides", tc.apiserverArgOverrides)
cmd.Flags().Set("controllermanager-arg-overrides", tc.cmArgOverrides)
if tc.storageBackend != "" {
cmd.Flags().Set("storage-backend", tc.storageBackend)
}
@ -259,6 +268,64 @@ func TestInitFederation(t *testing.T) {
}
}
func TestMarshallAndMergeOverrides(t *testing.T) {
testCases := []struct {
overrideParams string
expectedSet sets.String
expectedErr string
}{
{
overrideParams: "valid-format-param1=override1,valid-format-param2=override2",
expectedSet: sets.NewString("arg2=val2", "arg1=val1", "valid-format-param1=override1", "valid-format-param2=override2"),
expectedErr: "",
},
{
overrideParams: "valid-format-param1=override1,arg1=override1",
expectedSet: sets.NewString("arg2=val2", "arg1=override1", "valid-format-param1=override1"),
expectedErr: "",
},
{
overrideParams: "zero-value-arg=",
expectedSet: sets.NewString("arg2=val2", "arg1=val1", "zero-value-arg="),
expectedErr: "",
},
{
overrideParams: "wrong-format-arg",
expectedErr: "wrong format for override arg: wrong-format-arg",
},
{
overrideParams: "wrong-format-arg=override=wrong-format-arg=override",
expectedErr: "wrong format for override arg: wrong-format-arg=override=wrong-format-arg=override",
},
{
overrideParams: "=wrong-format-only-value",
expectedErr: "wrong format for override arg: =wrong-format-only-value, arg name cannot be empty",
},
}
for i, tc := range testCases {
args, err := marshallOverrides(tc.overrideParams)
if tc.expectedErr == "" {
origArgs := map[string]string{
"arg1": "val1",
"arg2": "val2",
}
merged := argMapsToArgStrings(origArgs, args)
got := sets.NewString(merged...)
want := tc.expectedSet
if !got.Equal(want) {
t.Errorf("[%d] unexpected output: got: %v, want: %v", i, got, want)
}
} else {
if err.Error() != tc.expectedErr {
t.Errorf("[%d] unexpected error output: got: %s, want: %s", i, err.Error(), tc.expectedErr)
}
}
}
}
// TestCertsTLS tests TLS handshake with client authentication for any server
// name. There is a separate test below to test the certificate generation
// end-to-end over HTTPS.
@ -498,7 +565,7 @@ func TestCertsHTTPS(t *testing.T) {
}
}
func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) {
func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider, apiserverOverrideArg, cmOverrideArg string) (cmdutil.Factory, error) {
svcName := federationName + "-apiserver"
svcUrlPrefix := "/api/v1/namespaces/federation-system/services"
credSecretName := svcName + "-credentials"
@ -684,6 +751,40 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
nodeList := v1.NodeList{}
nodeList.Items = append(nodeList.Items, node)
address := lbIp
if apiserverServiceType == v1.ServiceTypeNodePort {
if advertiseAddress != "" {
address = advertiseAddress
} else {
address = nodeIP
}
}
apiserverCommand := []string{
"/hyperkube",
"federation-apiserver",
}
apiserverArgs := []string{
"--bind-address=0.0.0.0",
"--etcd-servers=http://localhost:2379",
"--secure-port=443",
"--tls-cert-file=/etc/federation/apiserver/server.crt",
"--tls-private-key-file=/etc/federation/apiserver/server.key",
"--admission-control=NamespaceLifecycle",
fmt.Sprintf("--storage-backend=%s", storageProvider),
fmt.Sprintf("--advertise-address=%s", address),
}
if apiserverOverrideArg != "" {
apiserverArgs = append(apiserverArgs, "--client-ca-file=override")
apiserverArgs = append(apiserverArgs, "--log-dir=override")
} else {
apiserverArgs = append(apiserverArgs, "--client-ca-file=/etc/federation/apiserver/ca.crt")
}
sort.Strings(apiserverArgs)
apiserverCommand = append(apiserverCommand, apiserverArgs...)
apiserver := v1beta1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
@ -707,18 +808,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
{
Name: "apiserver",
Image: image,
Command: []string{
"/hyperkube",
"federation-apiserver",
"--bind-address=0.0.0.0",
"--etcd-servers=http://localhost:2379",
"--secure-port=443",
"--client-ca-file=/etc/federation/apiserver/ca.crt",
"--tls-cert-file=/etc/federation/apiserver/server.crt",
"--tls-private-key-file=/etc/federation/apiserver/server.key",
"--admission-control=NamespaceLifecycle",
fmt.Sprintf("--storage-backend=%s", storageProvider),
},
Command: apiserverCommand,
Ports: []v1.ContainerPort{
{
Name: "https",
@ -784,15 +874,28 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
}
}
address := lbIp
if apiserverServiceType == v1.ServiceTypeNodePort {
if advertiseAddress != "" {
address = advertiseAddress
cmCommand := []string{
"/hyperkube",
"federation-controller-manager",
}
cmArgs := []string{
"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
"--dns-provider-config=",
fmt.Sprintf("--federation-name=%s", federationName),
fmt.Sprintf("--zone-name=%s", dnsZoneName),
fmt.Sprintf("--master=https://%s", svcName),
}
if cmOverrideArg != "" {
cmArgs = append(cmArgs, "--dns-provider=override")
cmArgs = append(cmArgs, "--log-dir=override")
} else {
address = nodeIP
cmArgs = append(cmArgs, fmt.Sprintf("--dns-provider=%s", dnsProvider))
}
}
apiserver.Spec.Template.Spec.Containers[0].Command = append(apiserver.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--advertise-address=%s", address))
sort.Strings(cmArgs)
cmCommand = append(cmCommand, cmArgs...)
cmName := federationName + "-controller-manager"
cm := v1beta1.Deployment{
@ -818,16 +921,7 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
{
Name: "controller-manager",
Image: image,
Command: []string{
"/hyperkube",
"federation-controller-manager",
"--master=https://" + svcName,
"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
fmt.Sprintf("--dns-provider=%s", dnsProvider),
"--dns-provider-config=",
fmt.Sprintf("--federation-name=%s", federationName),
fmt.Sprintf("--zone-name=%s", dnsZoneName),
},
Command: cmCommand,
VolumeMounts: []v1.VolumeMount{
{
Name: cmKubeconfigSecretName,

2
hack/verify-flags/known-flags.txt
View File

@ -23,6 +23,7 @@ api-server-advertise-address
api-server-service-type
api-token
api-version
apiserver-arg-overrides
apiserver-count
apiserver-count
audit-log-maxage
@ -114,6 +115,7 @@ container-runtime
container-runtime-endpoint
contain-pod-resources
contention-profiling
controllermanager-arg-overrides
controller-start-interval
cors-allowed-origins
cpu-cfs-quota