diff --git a/cmd/kubeadm/app/apis/kubeadm/argument.go b/cmd/kubeadm/app/apis/kubeadm/argument.go new file mode 100644 index 00000000000..91d373c8ee4 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/argument.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +// GetArgValue traverses an argument slice backwards and returns the value +// of the given argument name and the index where it was found. +// If the argument does not exist an empty string and -1 are returned. +// startIdx defines where the iteration starts. If startIdx is a negative +// value or larger than the size of the argument slice the iteration +// will start from the last element. +func GetArgValue(args []Arg, name string, startIdx int) (string, int) { + if startIdx < 0 || startIdx > len(args)-1 { + startIdx = len(args) - 1 + } + for i := startIdx; i >= 0; i-- { + arg := args[i] + if arg.Name == name { + return arg.Value, i + } + } + return "", -1 +} + +// SetArgValues updates the value of one or more arguments or adds a new +// one if missing. The function works backwards in the argument list. +// nArgs holds how many existing arguments with this name should be set. +// If nArgs is less than 1, all of them will be updated. +func SetArgValues(args []Arg, name, value string, nArgs int) []Arg { + var count int + var found bool + for i := len(args) - 1; i >= 0; i-- { + if args[i].Name == name { + found = true + args[i].Value = value + if nArgs < 1 { + continue + } + count++ + if count >= nArgs { + return args + } + } + } + if found { + return args + } + args = append(args, Arg{Name: name, Value: value}) + return args +} diff --git a/cmd/kubeadm/app/apis/kubeadm/argument_test.go b/cmd/kubeadm/app/apis/kubeadm/argument_test.go new file mode 100644 index 00000000000..fcb523e4daa --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/argument_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +import ( + "reflect" + "testing" +) + +func TestGetArgValue(t *testing.T) { + var tests = []struct { + testName string + args []Arg + name string + expectedValue string + startIdx int + expectedIdx int + }{ + { + testName: "argument exists with non-empty value", + args: []Arg{{Name: "a", Value: "a1"}, {Name: "b", Value: "b1"}, {Name: "c", Value: "c1"}}, + name: "b", + expectedValue: "b1", + expectedIdx: 1, + startIdx: -1, + }, + { + testName: "argument exists with non-empty value (offset index)", + args: []Arg{{Name: "a", Value: "a1"}, {Name: "b", Value: "b1"}, {Name: "c", Value: "c1"}}, + name: "a", + expectedValue: "a1", + expectedIdx: 0, + startIdx: 0, + }, + { + testName: "argument exists with empty value", + args: []Arg{{Name: "foo1", Value: ""}, {Name: "foo2", Value: ""}}, + name: "foo2", + expectedValue: "", + expectedIdx: 1, + startIdx: -1, + }, + { + testName: "argument does not exists", + args: []Arg{{Name: "foo", Value: "bar"}}, + name: "z", + expectedValue: "", + expectedIdx: -1, + startIdx: -1, + }, + } + + for _, rt := range tests { + t.Run(rt.testName, func(t *testing.T) { + value, idx := GetArgValue(rt.args, rt.name, rt.startIdx) + if idx != rt.expectedIdx { + t.Errorf("expected index: %v, got: %v", rt.expectedIdx, idx) + } + if value != rt.expectedValue { + t.Errorf("expected value: %s, got: %s", rt.expectedValue, value) + } + }) + } +} + +func TestSetArgValues(t *testing.T) { + var tests = []struct { + testName string + args []Arg + name string + value string + nArgs int + expectedArgs []Arg + }{ + { + testName: "update 1 argument", + args: []Arg{{Name: "foo", Value: "bar1"}, {Name: "foo", Value: "bar2"}}, + name: "foo", + value: "zz", + nArgs: 1, + expectedArgs: []Arg{{Name: "foo", Value: "bar1"}, {Name: "foo", Value: "zz"}}, + }, + { + testName: "update all arguments", + args: []Arg{{Name: "foo", Value: "bar1"}, {Name: "foo", Value: "bar2"}}, + name: "foo", + value: "zz", + nArgs: -1, + expectedArgs: []Arg{{Name: "foo", Value: "zz"}, {Name: "foo", Value: "zz"}}, + }, + { + testName: "add new argument", + args: []Arg{{Name: "foo", Value: "bar1"}, {Name: "foo", Value: "bar2"}}, + name: "z", + value: "zz", + nArgs: -1, + expectedArgs: []Arg{{Name: "foo", Value: "bar1"}, {Name: "foo", Value: "bar2"}, {Name: "z", Value: "zz"}}, + }, + } + + for _, rt := range tests { + t.Run(rt.testName, func(t *testing.T) { + args := SetArgValues(rt.args, rt.name, rt.value, rt.nArgs) + if !reflect.DeepEqual(args, rt.expectedArgs) { + t.Errorf("expected args: %#v, got: %#v", rt.expectedArgs, args) + } + }) + } +} diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 63c7285cd03..8097a3497cc 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -145,11 +145,10 @@ type ClusterConfiguration struct { // ControlPlaneComponent holds settings common to control plane component of the cluster type ControlPlaneComponent struct { // ExtraArgs is an extra set of flags to pass to the control plane component. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. - ExtraArgs map[string]string + // An argument name in this list is the flag name as it appears on the + // command line except without leading dash(es). Extra arguments will override existing + // default arguments. Duplicate extra arguments are allowed. + ExtraArgs []Arg // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. ExtraVolumes []HostPathMount @@ -220,9 +219,9 @@ type NodeRegistrationOptions struct { // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config ConfigMap // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - KubeletExtraArgs map[string]string + // An argument name in this list is the flag name as it appears on the command line except without leading dash(es). + // Extra arguments will override existing default arguments. Duplicate extra arguments are allowed. + KubeletExtraArgs []Arg // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered, e.g. 'IsPrivilegedUser,Swap'. // Value 'all' ignores errors from all checks. @@ -267,9 +266,10 @@ type LocalEtcd struct { // ExtraArgs are extra arguments provided to the etcd binary // when run inside a static pod. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - ExtraArgs map[string]string + // An argument name in this list is the flag name as it appears on the + // command line except without leading dash(es). Extra arguments will override existing + // default arguments. Duplicate extra arguments are allowed. + ExtraArgs []Arg // ExtraEnvs is an extra set of environment variables to pass to the control plane component. // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. @@ -505,3 +505,9 @@ type ResetConfiguration struct { // ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig type ComponentConfigMap map[string]ComponentConfig + +// Arg represents an argument with a name and a value. +type Arg struct { + Name string + Value string +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go index 8e0c079809c..1b19007104e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta3 import ( + "sort" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/conversion" @@ -43,19 +45,65 @@ func Convert_v1beta3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConf // Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent is required due to the missing ControlPlaneComponent.ExtraEnvs in v1beta3. func Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { out.ExtraEnvs = []v1.EnvVar{} + out.ExtraArgs = convertToArgs(in.ExtraArgs) return autoConvert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in, out, s) } func Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { + out.ExtraArgs = convertFromArgs(in.ExtraArgs) return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in, out, s) } // Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd is required due to the missing LocalEtcd.ExtraEnvs in v1beta3. func Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { out.ExtraEnvs = []v1.EnvVar{} + out.ExtraArgs = convertToArgs(in.ExtraArgs) return autoConvert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) } func Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { + out.ExtraArgs = convertFromArgs(in.ExtraArgs) return autoConvert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in, out, s) } + +func Convert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { + out.KubeletExtraArgs = convertToArgs(in.KubeletExtraArgs) + return autoConvert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in, out, s) +} + +func Convert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { + out.KubeletExtraArgs = convertFromArgs(in.KubeletExtraArgs) + return autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(in, out, s) +} + +// convertToArgs takes a argument map and converts it to a slice of arguments. +// Te resulting argument slice is sorted alpha-numerically. +func convertToArgs(in map[string]string) []kubeadm.Arg { + if in == nil { + return nil + } + args := make([]kubeadm.Arg, 0, len(in)) + for k, v := range in { + args = append(args, kubeadm.Arg{Name: k, Value: v}) + } + sort.Slice(args, func(i, j int) bool { + if args[i].Name == args[j].Name { + return args[i].Value < args[j].Value + } + return args[i].Name < args[j].Name + }) + return args +} + +// convertFromArgs takes a slice of arguments and returns an argument map. +// Duplicate argument keys will be de-duped, where later keys will take precedence. +func convertFromArgs(in []kubeadm.Arg) map[string]string { + if in == nil { + return nil + } + args := make(map[string]string, len(in)) + for _, arg := range in { + args[arg.Name] = arg.Value + } + return args +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion_test.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion_test.go new file mode 100644 index 00000000000..ad861bef02b --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta3 + +import ( + "reflect" + "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +func TestConvertToArgs(t *testing.T) { + var tests = []struct { + name string + args map[string]string + expectedArgs []kubeadmapi.Arg + }{ + { + name: "nil map returns nil args", + args: nil, + expectedArgs: nil, + }, + { + name: "valid args are parsed (sorted)", + args: map[string]string{"c": "d", "a": "b"}, + expectedArgs: []kubeadmapi.Arg{ + {Name: "a", Value: "b"}, + {Name: "c", Value: "d"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := convertToArgs(tc.args) + if !reflect.DeepEqual(tc.expectedArgs, actual) { + t.Errorf("expected args: %v\n\t got: %v\n\t", tc.expectedArgs, actual) + } + }) + } +} + +func TestConvertFromArgs(t *testing.T) { + var tests = []struct { + name string + args []kubeadmapi.Arg + expectedArgs map[string]string + }{ + { + name: "nil args return nil map", + args: nil, + expectedArgs: nil, + }, + { + name: "valid args are parsed", + args: []kubeadmapi.Arg{ + {Name: "a", Value: "b"}, + {Name: "c", Value: "d"}, + }, + expectedArgs: map[string]string{"a": "b", "c": "d"}, + }, + { + name: "duplicates are dropped", + args: []kubeadmapi.Arg{ + {Name: "a", Value: "b"}, + {Name: "c", Value: "d1"}, + {Name: "c", Value: "d2"}, + }, + expectedArgs: map[string]string{"a": "b", "c": "d2"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := convertFromArgs(tc.args) + if !reflect.DeepEqual(tc.expectedArgs, actual) { + t.Errorf("expected args: %v\n\t got: %v\n\t", tc.expectedArgs, actual) + } + }) + } +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go index 881434d2ae1..7eb0fdf209b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go @@ -174,16 +174,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*NodeRegistrationOptions)(nil), (*kubeadm.NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(a.(*NodeRegistrationOptions), b.(*kubeadm.NodeRegistrationOptions), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.NodeRegistrationOptions)(nil), (*NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(a.(*kubeadm.NodeRegistrationOptions), b.(*NodeRegistrationOptions), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Patches)(nil), (*kubeadm.Patches)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_Patches_To_kubeadm_Patches(a.(*Patches), b.(*kubeadm.Patches), scope) }); err != nil { @@ -214,6 +204,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.NodeRegistrationOptions)(nil), (*NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(a.(*kubeadm.NodeRegistrationOptions), b.(*NodeRegistrationOptions), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*ControlPlaneComponent)(nil), (*kubeadm.ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(a.(*ControlPlaneComponent), b.(*kubeadm.ControlPlaneComponent), scope) }); err != nil { @@ -229,6 +224,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*NodeRegistrationOptions)(nil), (*kubeadm.NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(a.(*NodeRegistrationOptions), b.(*kubeadm.NodeRegistrationOptions), scope) + }); err != nil { + return err + } return nil } @@ -378,13 +378,13 @@ func Convert_kubeadm_ClusterConfiguration_To_v1beta3_ClusterConfiguration(in *ku } func autoConvert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + // WARNING: in.ExtraArgs requires manual conversion: inconvertible types (map[string]string vs []k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg) out.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) return nil } func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + // WARNING: in.ExtraArgs requires manual conversion: inconvertible types ([]k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg vs map[string]string) out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) // WARNING: in.ExtraEnvs requires manual conversion: does not exist in peer-type return nil @@ -669,7 +669,7 @@ func autoConvert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kube return err } out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + // WARNING: in.ExtraArgs requires manual conversion: inconvertible types (map[string]string vs []k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil @@ -680,7 +680,7 @@ func autoConvert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in *kubeadm.LocalEtcd, o return err } out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + // WARNING: in.ExtraArgs requires manual conversion: inconvertible types ([]k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg vs map[string]string) // WARNING: in.ExtraEnvs requires manual conversion: does not exist in peer-type out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) @@ -715,32 +715,22 @@ func autoConvert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOpti out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + // WARNING: in.KubeletExtraArgs requires manual conversion: inconvertible types (map[string]string vs []k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg) out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy) return nil } -// Convert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_v1beta3_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in, out, s) -} - func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + // WARNING: in.KubeletExtraArgs requires manual conversion: inconvertible types ([]k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm.Arg vs map[string]string) out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy) return nil } -// Convert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta3_NodeRegistrationOptions(in, out, s) -} - func autoConvert_v1beta3_Patches_To_kubeadm_Patches(in *Patches, out *kubeadm.Patches, s conversion.Scope) error { out.Directory = in.Directory return nil diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go index f87f92f83ce..511d946e72c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go @@ -26,9 +26,12 @@ limitations under the License. // // - TODO https://github.com/kubernetes/kubeadm/issues/2890 // - Support custom environment variables in control plane components under `ClusterConfiguration`. -// Use `APIServer.ExtraEnvs`, `ControllerManager.ExtraEnvs`, `Scheduler.ExtraEnvs`, `Etcd.Local.ExtraEnvs`. +// Use `APIServer.ExtraEnvs`, `ControllerManager.ExtraEnvs`, `Scheduler.ExtraEnvs`, `Etcd.Local.ExtraEnvs`. // - The ResetConfiguration API type is now supported in v1beta4. Users are able to reset a node by passing a --config file to "kubeadm reset". // - `dry-run` mode in is now configurable in InitConfiguration and JoinConfiguration config files. +// - Replace the existing string/string extra argument maps with structured extra arguments that support duplicates. +// The change applies to `ClusterConfiguration` - `APIServer.ExtraArgs, `ControllerManager.ExtraArgs`, +// `Scheduler.ExtraArgs`, `Etcd.Local.ExtraArgs`. Also to `NodeRegistrationOptions.KubeletExtraArgs`. // // Migration from old kubeadm config versions // diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go index b91863971ae..48562e6c562 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go @@ -144,12 +144,11 @@ type ClusterConfiguration struct { // ControlPlaneComponent holds settings common to control plane component of the cluster type ControlPlaneComponent struct { // ExtraArgs is an extra set of flags to pass to the control plane component. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. + // An argument name in this list is the flag name as it appears on the + // command line except without leading dash(es). Extra arguments will override existing + // default arguments. Duplicate extra arguments are allowed. // +optional - ExtraArgs map[string]string `json:"extraArgs,omitempty"` + ExtraArgs []Arg `json:"extraArgs,omitempty"` // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. // +optional @@ -232,10 +231,10 @@ type NodeRegistrationOptions struct { // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config ConfigMap // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). + // An argument name in this list is the flag name as it appears on the command line except without leading dash(es). + // Extra arguments will override existing default arguments. Duplicate extra arguments are allowed. // +optional - KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` + KubeletExtraArgs []Arg `json:"kubeletExtraArgs,omitempty"` // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered, e.g. 'IsPrivilegedUser,Swap'. // Value 'all' ignores errors from all checks. @@ -287,10 +286,11 @@ type LocalEtcd struct { // ExtraArgs are extra arguments provided to the etcd binary // when run inside a static pod. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). + // An argument name in this list is the flag name as it appears on the + // command line except without leading dash(es). Extra arguments will override existing + // default arguments. Duplicate extra arguments are allowed. // +optional - ExtraArgs map[string]string `json:"extraArgs,omitempty"` + ExtraArgs []Arg `json:"extraArgs,omitempty"` // ExtraEnvs is an extra set of environment variables to pass to the control plane component. // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. @@ -500,3 +500,9 @@ type ResetConfiguration struct { // +optional SkipPhases []string `json:"skipPhases,omitempty"` } + +// Arg represents an argument with a name and a value. +type Arg struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go index 6db00601463..a203ddf12bb 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go @@ -59,6 +59,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*Arg)(nil), (*kubeadm.Arg)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta4_Arg_To_kubeadm_Arg(a.(*Arg), b.(*kubeadm.Arg), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kubeadm.Arg)(nil), (*Arg)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_Arg_To_v1beta4_Arg(a.(*kubeadm.Arg), b.(*Arg), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*BootstrapTokenDiscovery)(nil), (*kubeadm.BootstrapTokenDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta4_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(a.(*BootstrapTokenDiscovery), b.(*kubeadm.BootstrapTokenDiscovery), scope) }); err != nil { @@ -292,6 +302,28 @@ func Convert_kubeadm_APIServer_To_v1beta4_APIServer(in *kubeadm.APIServer, out * return autoConvert_kubeadm_APIServer_To_v1beta4_APIServer(in, out, s) } +func autoConvert_v1beta4_Arg_To_kubeadm_Arg(in *Arg, out *kubeadm.Arg, s conversion.Scope) error { + out.Name = in.Name + out.Value = in.Value + return nil +} + +// Convert_v1beta4_Arg_To_kubeadm_Arg is an autogenerated conversion function. +func Convert_v1beta4_Arg_To_kubeadm_Arg(in *Arg, out *kubeadm.Arg, s conversion.Scope) error { + return autoConvert_v1beta4_Arg_To_kubeadm_Arg(in, out, s) +} + +func autoConvert_kubeadm_Arg_To_v1beta4_Arg(in *kubeadm.Arg, out *Arg, s conversion.Scope) error { + out.Name = in.Name + out.Value = in.Value + return nil +} + +// Convert_kubeadm_Arg_To_v1beta4_Arg is an autogenerated conversion function. +func Convert_kubeadm_Arg_To_v1beta4_Arg(in *kubeadm.Arg, out *Arg, s conversion.Scope) error { + return autoConvert_kubeadm_Arg_To_v1beta4_Arg(in, out, s) +} + func autoConvert_v1beta4_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in *BootstrapTokenDiscovery, out *kubeadm.BootstrapTokenDiscovery, s conversion.Scope) error { out.Token = in.Token out.APIServerEndpoint = in.APIServerEndpoint @@ -388,7 +420,7 @@ func Convert_kubeadm_ClusterConfiguration_To_v1beta4_ClusterConfiguration(in *ku } func autoConvert_v1beta4_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraArgs = *(*[]kubeadm.Arg)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) return nil @@ -400,7 +432,7 @@ func Convert_v1beta4_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in * } func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta4_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraArgs = *(*[]Arg)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) return nil @@ -681,7 +713,7 @@ func autoConvert_v1beta4_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kube return err } out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraArgs = *(*[]kubeadm.Arg)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) @@ -698,7 +730,7 @@ func autoConvert_kubeadm_LocalEtcd_To_v1beta4_LocalEtcd(in *kubeadm.LocalEtcd, o return err } out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraArgs = *(*[]Arg)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) @@ -738,7 +770,7 @@ func autoConvert_v1beta4_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOpti out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + out.KubeletExtraArgs = *(*[]kubeadm.Arg)(unsafe.Pointer(&in.KubeletExtraArgs)) out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy) return nil @@ -753,7 +785,7 @@ func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta4_NodeRegistrationOpti out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + out.KubeletExtraArgs = *(*[]Arg)(unsafe.Pointer(&in.KubeletExtraArgs)) out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy) return nil diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go index 30e337cea90..c030a560dc1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go @@ -71,6 +71,22 @@ func (in *APIServer) DeepCopy() *APIServer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Arg) DeepCopyInto(out *Arg) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Arg. +func (in *Arg) DeepCopy() *Arg { + if in == nil { + return nil + } + out := new(Arg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapTokenDiscovery) DeepCopyInto(out *BootstrapTokenDiscovery) { *out = *in @@ -135,10 +151,8 @@ func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { *out = *in if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.ExtraVolumes != nil { in, out := &in.ExtraVolumes, &out.ExtraVolumes @@ -417,10 +431,8 @@ func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { out.ImageMeta = in.ImageMeta if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.ExtraEnvs != nil { in, out := &in.ExtraEnvs, &out.ExtraEnvs @@ -480,10 +492,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { } if in.KubeletExtraArgs != nil { in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.IgnorePreflightErrors != nil { in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index bd0c32a1f8f..f03f3471a38 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -65,6 +65,8 @@ func ValidateClusterConfiguration(c *kubeadm.ClusterConfiguration) field.ErrorLi allErrs = append(allErrs, ValidateDNS(&c.DNS, field.NewPath("dns"))...) allErrs = append(allErrs, ValidateNetworking(c, field.NewPath("networking"))...) allErrs = append(allErrs, ValidateAPIServer(&c.APIServer, field.NewPath("apiServer"))...) + allErrs = append(allErrs, ValidateControllerManager(&c.ControllerManager, field.NewPath("controllerManager"))...) + allErrs = append(allErrs, ValidateScheduler(&c.Scheduler, field.NewPath("scheduler"))...) allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...) allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("featureGates"))...) allErrs = append(allErrs, ValidateHostPort(c.ControlPlaneEndpoint, field.NewPath("controlPlaneEndpoint"))...) @@ -78,6 +80,21 @@ func ValidateClusterConfiguration(c *kubeadm.ClusterConfiguration) field.ErrorLi func ValidateAPIServer(a *kubeadm.APIServer, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, ValidateCertSANs(a.CertSANs, fldPath.Child("certSANs"))...) + allErrs = append(allErrs, ValidateExtraArgs(a.ExtraArgs, fldPath.Child("extraArgs"))...) + return allErrs +} + +// ValidateControllerManager validates the controller manager object and collects all encountered errors +func ValidateControllerManager(a *kubeadm.ControlPlaneComponent, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, ValidateExtraArgs(a.ExtraArgs, fldPath.Child("extraArgs"))...) + return allErrs +} + +// ValidateScheduler validates the scheduler object and collects all encountered errors +func ValidateScheduler(a *kubeadm.ControlPlaneComponent, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, ValidateExtraArgs(a.ExtraArgs, fldPath.Child("extraArgs"))...) return allErrs } @@ -116,6 +133,7 @@ func ValidateNodeRegistrationOptions(nro *kubeadm.NodeRegistrationOptions, fldPa } } allErrs = append(allErrs, ValidateSocketPath(nro.CRISocket, fldPath.Child("criSocket"))...) + allErrs = append(allErrs, ValidateExtraArgs(nro.KubeletExtraArgs, fldPath.Child("kubeletExtraArgs"))...) // TODO: Maybe validate .Taints as well in the future using something like validateNodeTaints() in pkg/apis/core/validation return allErrs } @@ -288,6 +306,7 @@ func ValidateEtcd(e *kubeadm.Etcd, fldPath *field.Path) field.ErrorList { if len(e.Local.ImageRepository) > 0 { allErrs = append(allErrs, ValidateImageRepository(e.Local.ImageRepository, localPath.Child("imageRepository"))...) } + allErrs = append(allErrs, ValidateExtraArgs(e.Local.ExtraArgs, localPath.Child("extraArgs"))...) } if e.External != nil { requireHTTPS := true @@ -479,9 +498,10 @@ func getClusterNodeMask(c *kubeadm.ClusterConfiguration, isIPv6 bool) (int, erro maskArg = "node-cidr-mask-size-ipv4" } - if v, ok := c.ControllerManager.ExtraArgs[maskArg]; ok && v != "" { + maskValue, _ := kubeadm.GetArgValue(c.ControllerManager.ExtraArgs, maskArg, -1) + if len(maskValue) != 0 { // assume it is an integer, if not it will fail later - maskSize, err = strconv.Atoi(v) + maskSize, err = strconv.Atoi(maskValue) if err != nil { return 0, errors.Wrapf(err, "could not parse the value of the kube-controller-manager flag %s as an integer", maskArg) } @@ -519,7 +539,8 @@ func ValidateNetworking(c *kubeadm.ClusterConfiguration, fldPath *field.Path) fi } if len(c.Networking.PodSubnet) != 0 { allErrs = append(allErrs, ValidateIPNetFromString(c.Networking.PodSubnet, constants.MinimumAddressesInPodSubnet, fldPath.Child("podSubnet"))...) - if c.ControllerManager.ExtraArgs["allocate-node-cidrs"] != "false" { + val, _ := kubeadm.GetArgValue(c.ControllerManager.ExtraArgs, "allocate-node-cidrs", -1) + if val != "false" { // Pod subnet was already validated, we need to validate now against the node-mask allErrs = append(allErrs, ValidatePodSubnetNodeMask(c.Networking.PodSubnet, c, fldPath.Child("podSubnet"))...) } @@ -676,3 +697,16 @@ func ValidateResetConfiguration(c *kubeadm.ResetConfiguration) field.ErrorList { allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...) return allErrs } + +// ValidateExtraArgs validates a set of arguments and collects all encountered errors +func ValidateExtraArgs(args []kubeadm.Arg, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + for idx, arg := range args { + if len(arg.Name) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath, fmt.Sprintf("index %d", idx), "argument has no name")) + } + } + + return allErrs +} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index e41f867c9dd..c6ddf38e72b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -254,24 +254,24 @@ func TestValidatePodSubnetNodeMask(t *testing.T) { var tests = []struct { name string subnet string - cmExtraArgs map[string]string + cmExtraArgs []kubeadmapi.Arg expected bool }{ // dual-stack: {"dual IPv4 only, but mask too small. Default node-mask", "10.0.0.16/29", nil, false}, - {"dual IPv4 only, but mask too small. Configured node-mask", "10.0.0.16/24", map[string]string{"node-cidr-mask-size-ipv4": "23"}, false}, + {"dual IPv4 only, but mask too small. Configured node-mask", "10.0.0.16/24", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv4", Value: "23"}}, false}, {"dual IPv6 only, but mask too small. Default node-mask", "2001:db8::1/112", nil, false}, - {"dual IPv6 only, but mask too small. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "24"}, false}, + {"dual IPv6 only, but mask too small. Configured node-mask", "2001:db8::1/64", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv6", Value: "24"}}, false}, {"dual IPv6 only, but mask difference greater than 16. Default node-mask", "2001:db8::1/12", nil, false}, - {"dual IPv6 only, but mask difference greater than 16. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "120"}, false}, + {"dual IPv6 only, but mask difference greater than 16. Configured node-mask", "2001:db8::1/64", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv6", Value: "120"}}, false}, {"dual IPv4 only CIDR", "10.0.0.16/12", nil, true}, {"dual IPv6 only CIDR", "2001:db8::/48", nil, true}, {"dual, but IPv4 mask too small. Default node-mask", "10.0.0.16/29,2001:db8::/48", nil, false}, - {"dual, but IPv4 mask too small. Configured node-mask", "10.0.0.16/24,2001:db8::/48", map[string]string{"node-cidr-mask-size-ipv4": "23"}, false}, + {"dual, but IPv4 mask too small. Configured node-mask", "10.0.0.16/24,2001:db8::/48", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv4", Value: "23"}}, false}, {"dual, but IPv6 mask too small. Default node-mask", "2001:db8::1/112,10.0.0.16/16", nil, false}, - {"dual, but IPv6 mask too small. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "24"}, false}, + {"dual, but IPv6 mask too small. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv6", Value: "24"}}, false}, {"dual, but mask difference greater than 16. Default node-mask", "2001:db8::1/12,10.0.0.16/16", nil, false}, - {"dual, but mask difference greater than 16. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "120"}, false}, + {"dual, but mask difference greater than 16. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", []kubeadmapi.Arg{{Name: "node-cidr-mask-size-ipv6", Value: "120"}}, false}, {"dual IPv4 IPv6", "2001:db8::/48,10.0.0.16/12", nil, true}, {"dual IPv6 IPv4", "2001:db8::/48,10.0.0.16/12", nil, true}, } @@ -1205,7 +1205,10 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv4 custom mask", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "21", "node-cidr-mask-size-ipv4": "23"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size", Value: "21"}, + {Name: "node-cidr-mask-size-ipv4", Value: "23"}, + }, }, }, isIPv6: false, @@ -1221,7 +1224,9 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv6 custom mask", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv6": "83"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size-ipv6", Value: "83"}, + }, }, }, isIPv6: true, @@ -1231,7 +1236,9 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv4 custom mask", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "23"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size-ipv4", Value: "23"}, + }, }, }, isIPv6: false, @@ -1241,7 +1248,9 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv4 wrong mask", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "aa"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size-ipv4", Value: "aa"}, + }, }, }, isIPv6: false, @@ -1251,7 +1260,9 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv6 default mask and legacy flag", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size", Value: "23"}, + }, }, }, isIPv6: true, @@ -1261,7 +1272,10 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv6 custom mask and legacy flag", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23", "node-cidr-mask-size-ipv6": "83"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size", Value: "23"}, + {Name: "node-cidr-mask-size-ipv6", Value: "83"}, + }, }, }, isIPv6: true, @@ -1271,7 +1285,10 @@ func TestGetClusterNodeMask(t *testing.T) { name: "dual ipv6 custom mask and wrong flag", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23", "node-cidr-mask-size-ipv6": "a83"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size", Value: "23"}, + {Name: "node-cidr-mask-size-ipv6", Value: "a83"}, + }, }, }, isIPv6: true, @@ -1390,3 +1407,34 @@ func TestValidateAbsolutePath(t *testing.T) { } } } + +func TestValidateExtraArgs(t *testing.T) { + var tests = []struct { + name string + args []kubeadmapi.Arg + expectedErrors int + }{ + { + name: "valid argument", + args: []kubeadmapi.Arg{{Name: "foo", Value: "bar"}}, + expectedErrors: 0, + }, + { + name: "invalid one argument", + args: []kubeadmapi.Arg{{Name: "", Value: "bar"}}, + expectedErrors: 1, + }, + { + name: "invalid two arguments", + args: []kubeadmapi.Arg{{Name: "", Value: "foo"}, {Name: "", Value: "bar"}}, + expectedErrors: 2, + }, + } + + for _, tc := range tests { + actual := ValidateExtraArgs(tc.args, nil) + if len(actual) != tc.expectedErrors { + t.Errorf("case %q:\n\t expected errors: %v\n\t got: %v\n\t errors: %v", tc.name, tc.expectedErrors, len(actual), actual) + } + } +} diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 3ab74104324..1e7d371323f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -71,6 +71,22 @@ func (in *APIServer) DeepCopy() *APIServer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Arg) DeepCopyInto(out *Arg) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Arg. +func (in *Arg) DeepCopy() *Arg { + if in == nil { + return nil + } + out := new(Arg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapTokenDiscovery) DeepCopyInto(out *BootstrapTokenDiscovery) { *out = *in @@ -164,10 +180,8 @@ func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { *out = *in if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.ExtraVolumes != nil { in, out := &in.ExtraVolumes, &out.ExtraVolumes @@ -447,10 +461,8 @@ func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { out.ImageMeta = in.ImageMeta if in.ExtraArgs != nil { in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.ExtraEnvs != nil { in, out := &in.ExtraEnvs, &out.ExtraEnvs @@ -510,10 +522,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { } if in.KubeletExtraArgs != nil { in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + *out = make([]Arg, len(*in)) + copy(*out, *in) } if in.IgnorePreflightErrors != nil { in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go index 19436dcba9a..37536b6e520 100644 --- a/cmd/kubeadm/app/cmd/options/generic.go +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -54,6 +54,9 @@ func AddIgnorePreflightErrorsFlag(fs *pflag.FlagSet, ignorePreflightErrors *[]st // AddControlPlanExtraArgsFlags adds the ExtraArgs flags for control plane components func AddControlPlanExtraArgsFlags(fs *pflag.FlagSet, apiServerExtraArgs, controllerManagerExtraArgs, schedulerExtraArgs *map[string]string) { + // TODO: https://github.com/kubernetes/kubeadm/issues/1601 + // Either deprecate these flags or handle duplicate keys. + // Currently the map[string]string returned by NewMapStringString() doesn't allow this. fs.Var(cliflag.NewMapStringString(apiServerExtraArgs), APIServerExtraArgs, "A set of extra flags to pass to the API Server or override default ones in form of =") fs.Var(cliflag.NewMapStringString(controllerManagerExtraArgs), ControllerManagerExtraArgs, "A set of extra flags to pass to the Controller Manager or override default ones in form of =") fs.Var(cliflag.NewMapStringString(schedulerExtraArgs), SchedulerExtraArgs, "A set of extra flags to pass to the Scheduler or override default ones in form of =") diff --git a/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go b/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go index 77a3487f70b..db72b541e83 100644 --- a/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go +++ b/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" @@ -78,8 +79,8 @@ func runKubeletFinalizeCertRotation(c workflow.RunData) error { // If yes, use that path, else use the kubeadm provided value. cfg := data.Cfg() pkiPath := filepath.Join(data.KubeletDir(), "pki") - val, ok := cfg.NodeRegistration.KubeletExtraArgs["cert-dir"] - if ok { + val, idx := kubeadmapi.GetArgValue(cfg.NodeRegistration.KubeletExtraArgs, "cert-dir", -1) + if idx > -1 { pkiPath = val } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 42725d8dab8..7096c1e79db 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -162,46 +162,47 @@ func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.Cluste // getAPIServerCommand builds the right API server command from the given config object and version func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint) []string { - defaultArguments := map[string]string{ - "advertise-address": localAPIEndpoint.AdvertiseAddress, - "enable-admission-plugins": "NodeRestriction", - "service-cluster-ip-range": cfg.Networking.ServiceSubnet, - "service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName), - "service-account-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), - "service-account-issuer": fmt.Sprintf("https://kubernetes.default.svc.%s", cfg.Networking.DNSDomain), - "client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), - "tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName), - "tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName), - "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), - "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), - "enable-bootstrap-token-auth": "true", - "secure-port": fmt.Sprintf("%d", localAPIEndpoint.BindPort), - "allow-privileged": "true", - "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + defaultArguments := []kubeadmapi.Arg{ + {Name: "advertise-address", Value: localAPIEndpoint.AdvertiseAddress}, + {Name: "enable-admission-plugins", Value: "NodeRestriction"}, + {Name: "service-cluster-ip-range", Value: cfg.Networking.ServiceSubnet}, + {Name: "service-account-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)}, + {Name: "service-account-signing-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)}, + {Name: "service-account-issuer", Value: fmt.Sprintf("https://kubernetes.default.svc.%s", cfg.Networking.DNSDomain)}, + {Name: "client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)}, + {Name: "tls-cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName)}, + {Name: "tls-private-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName)}, + {Name: "kubelet-client-certificate", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName)}, + {Name: "kubelet-client-key", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName)}, + {Name: "enable-bootstrap-token-auth", Value: "true"}, + {Name: "secure-port", Value: fmt.Sprintf("%d", localAPIEndpoint.BindPort)}, + {Name: "allow-privileged", Value: "true"}, + {Name: "kubelet-preferred-address-types", Value: "InternalIP,ExternalIP,Hostname"}, // add options to configure the front proxy. Without the generated client cert, this will never be useable // so add it unconditionally with recommended values - "requestheader-username-headers": "X-Remote-User", - "requestheader-group-headers": "X-Remote-Group", - "requestheader-extra-headers-prefix": "X-Remote-Extra-", - "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), - "requestheader-allowed-names": "front-proxy-client", - "proxy-client-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName), - "proxy-client-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName), + {Name: "requestheader-username-headers", Value: "X-Remote-User"}, + {Name: "requestheader-group-headers", Value: "X-Remote-Group"}, + {Name: "requestheader-extra-headers-prefix", Value: "X-Remote-Extra-"}, + {Name: "requestheader-client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)}, + {Name: "requestheader-allowed-names", Value: "front-proxy-client"}, + {Name: "proxy-client-cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)}, + {Name: "proxy-client-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)}, } command := []string{"kube-apiserver"} // If the user set endpoints for an external etcd cluster if cfg.Etcd.External != nil { - defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.External.Endpoints, ",") + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", strings.Join(cfg.Etcd.External.Endpoints, ","), 1) // Use any user supplied etcd certificates if cfg.Etcd.External.CAFile != "" { - defaultArguments["etcd-cafile"] = cfg.Etcd.External.CAFile + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-cafile", cfg.Etcd.External.CAFile, 1) } if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" { - defaultArguments["etcd-certfile"] = cfg.Etcd.External.CertFile - defaultArguments["etcd-keyfile"] = cfg.Etcd.External.KeyFile + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-certfile", cfg.Etcd.External.CertFile, 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-keyfile", cfg.Etcd.External.KeyFile, 1) + } } else { // Default to etcd static pod on localhost @@ -210,24 +211,25 @@ func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) { etcdLocalhostAddress = "::1" } - defaultArguments["etcd-servers"] = fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))) - defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName) - defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName) - defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))), 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-cafile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-certfile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName), 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-keyfile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName), 1) // Apply user configurations for local etcd if cfg.Etcd.Local != nil { - if value, ok := cfg.Etcd.Local.ExtraArgs["advertise-client-urls"]; ok { - defaultArguments["etcd-servers"] = value + if value, idx := kubeadmapi.GetArgValue(cfg.Etcd.Local.ExtraArgs, "advertise-client-urls", -1); idx > -1 { + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", value, 1) } } } if cfg.APIServer.ExtraArgs == nil { - cfg.APIServer.ExtraArgs = map[string]string{} + cfg.APIServer.ExtraArgs = []kubeadmapi.Arg{} } - cfg.APIServer.ExtraArgs["authorization-mode"] = getAuthzModes(cfg.APIServer.ExtraArgs["authorization-mode"]) - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServer.ExtraArgs)...) + authzVal, _ := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, "authorization-mode", -1) + cfg.APIServer.ExtraArgs = kubeadmapi.SetArgValues(cfg.APIServer.ExtraArgs, "authorization-mode", getAuthzModes(authzVal), 1) + command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.APIServer.ExtraArgs)...) return command } @@ -302,46 +304,46 @@ func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) caFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName) - defaultArguments := map[string]string{ - "bind-address": "127.0.0.1", - "leader-elect": "true", - "kubeconfig": kubeconfigFile, - "authentication-kubeconfig": kubeconfigFile, - "authorization-kubeconfig": kubeconfigFile, - "client-ca-file": caFile, - "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), - "root-ca-file": caFile, - "service-account-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), - "cluster-signing-cert-file": caFile, - "cluster-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName), - "use-service-account-credentials": "true", - "controllers": "*,bootstrapsigner,tokencleaner", + defaultArguments := []kubeadmapi.Arg{ + {Name: "bind-address", Value: "127.0.0.1"}, + {Name: "leader-elect", Value: "true"}, + {Name: "kubeconfig", Value: kubeconfigFile}, + {Name: "authentication-kubeconfig", Value: kubeconfigFile}, + {Name: "authorization-kubeconfig", Value: kubeconfigFile}, + {Name: "client-ca-file", Value: caFile}, + {Name: "requestheader-client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)}, + {Name: "root-ca-file", Value: caFile}, + {Name: "service-account-private-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)}, + {Name: "cluster-signing-cert-file", Value: caFile}, + {Name: "cluster-signing-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)}, + {Name: "use-service-account-credentials", Value: "true"}, + {Name: "controllers", Value: "*,bootstrapsigner,tokencleaner"}, } // If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path, // so that the csrsigning controller fails to start if res, _ := certphase.UsingExternalCA(cfg); res { - defaultArguments["cluster-signing-key-file"] = "" - defaultArguments["cluster-signing-cert-file"] = "" + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-signing-key-file", "", 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-signing-cert-file", "", 1) } // Let the controller-manager allocate Node CIDRs for the Pod network. // Each node will get a subspace of the address CIDR provided with --pod-network-cidr. if cfg.Networking.PodSubnet != "" { - defaultArguments["allocate-node-cidrs"] = "true" - defaultArguments["cluster-cidr"] = cfg.Networking.PodSubnet + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "allocate-node-cidrs", "true", 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-cidr", cfg.Networking.PodSubnet, 1) if cfg.Networking.ServiceSubnet != "" { - defaultArguments["service-cluster-ip-range"] = cfg.Networking.ServiceSubnet + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "service-cluster-ip-range", cfg.Networking.ServiceSubnet, 1) } } // Set cluster name if cfg.ClusterName != "" { - defaultArguments["cluster-name"] = cfg.ClusterName + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-name", cfg.ClusterName, 1) } command := []string{"kube-controller-manager"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManager.ExtraArgs)...) + command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.ControllerManager.ExtraArgs)...) return command } @@ -349,15 +351,15 @@ func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string // getSchedulerCommand builds the right scheduler command from the given config object and version func getSchedulerCommand(cfg *kubeadmapi.ClusterConfiguration) []string { kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) - defaultArguments := map[string]string{ - "bind-address": "127.0.0.1", - "leader-elect": "true", - "kubeconfig": kubeconfigFile, - "authentication-kubeconfig": kubeconfigFile, - "authorization-kubeconfig": kubeconfigFile, + defaultArguments := []kubeadmapi.Arg{ + {Name: "bind-address", Value: "127.0.0.1"}, + {Name: "leader-elect", Value: "true"}, + {Name: "kubeconfig", Value: kubeconfigFile}, + {Name: "authentication-kubeconfig", Value: kubeconfigFile}, + {Name: "authorization-kubeconfig", Value: kubeconfigFile}, } command := []string{"kube-scheduler"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Scheduler.ExtraArgs)...) + command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.Scheduler.ExtraArgs)...) return command } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 5ff2c1db285..78b09014d11 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -375,11 +375,11 @@ func TestGetAPIServerCommand(t *testing.T) { CertificatesDir: testCertsDir, APIServer: kubeadmapi.APIServer{ ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "service-cluster-ip-range": "baz", - "advertise-address": "9.9.9.9", - "audit-policy-file": "/etc/config/audit.yaml", - "audit-log-path": "/var/log/kubernetes", + ExtraArgs: []kubeadmapi.Arg{ + {Name: "service-cluster-ip-range", Value: "baz"}, + {Name: "advertise-address", Value: "9.9.9.9"}, + {Name: "audit-policy-file", Value: "/etc/config/audit.yaml"}, + {Name: "audit-log-path", Value: "/var/log/kubernetes"}, }, }, }, @@ -425,8 +425,8 @@ func TestGetAPIServerCommand(t *testing.T) { CertificatesDir: testCertsDir, APIServer: kubeadmapi.APIServer{ ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "authorization-mode": kubeadmconstants.ModeABAC, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "authorization-mode", Value: kubeadmconstants.ModeABAC}, }, }, }, @@ -470,12 +470,12 @@ func TestGetAPIServerCommand(t *testing.T) { CertificatesDir: testCertsDir, APIServer: kubeadmapi.APIServer{ ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "authorization-mode": strings.Join([]string{ + ExtraArgs: []kubeadmapi.Arg{ + {Name: "authorization-mode", Value: strings.Join([]string{ kubeadmconstants.ModeNode, kubeadmconstants.ModeRBAC, kubeadmconstants.ModeWebhook, - }, ","), + }, ",")}, }, }, }, @@ -660,7 +660,7 @@ func TestGetControllerManagerCommand(t *testing.T) { cfg: &kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16", DNSDomain: "cluster.local"}, ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "20"}, + ExtraArgs: []kubeadmapi.Arg{{Name: "node-cidr-mask-size", Value: "20"}}, }, CertificatesDir: testCertsDir, KubernetesVersion: cpVersion, @@ -726,7 +726,7 @@ func TestGetControllerManagerCommand(t *testing.T) { DNSDomain: "cluster.local", }, ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"allocate-node-cidrs": "false"}, + ExtraArgs: []kubeadmapi.Arg{{Name: "allocate-node-cidrs", Value: "false"}}, }, CertificatesDir: testCertsDir, KubernetesVersion: cpVersion, @@ -790,7 +790,10 @@ func TestGetControllerManagerCommand(t *testing.T) { DNSDomain: "cluster.local", }, ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "20", "node-cidr-mask-size-ipv6": "80"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "node-cidr-mask-size-ipv4", Value: "20"}, + {Name: "node-cidr-mask-size-ipv6", Value: "80"}, + }, }, CertificatesDir: testCertsDir, KubernetesVersion: cpVersion, diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index d193b4e0186..9aec46ce4df 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -72,8 +72,8 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.ClusterConfiguration) mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate) // Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec by default) // Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket. - flexvolumeDirVolumePath, ok := cfg.ControllerManager.ExtraArgs["flex-volume-plugin-dir"] - if !ok { + flexvolumeDirVolumePath, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, "flex-volume-plugin-dir", -1) + if idx == -1 { flexvolumeDirVolumePath = defaultFlexvolumeDirVolumePath } mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 129db183cc9..e58c6eb9ce7 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -239,31 +239,31 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) { etcdLocalhostAddress = "::1" } - defaultArguments := map[string]string{ - "name": nodeName, - // TODO: start using --initial-corrupt-check once the graduated flag is available: + defaultArguments := []kubeadmapi.Arg{ + {Name: "name", Value: nodeName}, + // TODO: start using --initial-corrupt-check once the graduated flag is available, // https://github.com/kubernetes/kubeadm/issues/2676 - "experimental-initial-corrupt-check": "true", - "listen-client-urls": fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)), - "advertise-client-urls": etcdutil.GetClientURL(endpoint), - "listen-peer-urls": etcdutil.GetPeerURL(endpoint), - "initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint), - "data-dir": cfg.Etcd.Local.DataDir, - "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), - "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), - "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), - "client-cert-auth": "true", - "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), - "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), - "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), - "peer-client-cert-auth": "true", - "snapshot-count": "10000", - "listen-metrics-urls": fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))), - "experimental-watch-progress-notify-interval": "5s", + {Name: "experimental-initial-corrupt-check", Value: "true"}, + {Name: "listen-client-urls", Value: fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint))}, + {Name: "advertise-client-urls", Value: etcdutil.GetClientURL(endpoint)}, + {Name: "listen-peer-urls", Value: etcdutil.GetPeerURL(endpoint)}, + {Name: "initial-advertise-peer-urls", Value: etcdutil.GetPeerURL(endpoint)}, + {Name: "data-dir", Value: cfg.Etcd.Local.DataDir}, + {Name: "cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName)}, + {Name: "key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName)}, + {Name: "trusted-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName)}, + {Name: "client-cert-auth", Value: "true"}, + {Name: "peer-cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName)}, + {Name: "peer-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName)}, + {Name: "peer-trusted-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName)}, + {Name: "peer-client-cert-auth", Value: "true"}, + {Name: "snapshot-count", Value: "10000"}, + {Name: "listen-metrics-urls", Value: fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort)))}, + {Name: "experimental-watch-progress-notify-interval", Value: "5s"}, } if len(initialCluster) == 0 { - defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint)) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "initial-cluster", fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint)), 1) } else { // NB. the joining etcd member should be part of the initialCluster list endpoints := []string{} @@ -271,12 +271,12 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL)) } - defaultArguments["initial-cluster"] = strings.Join(endpoints, ",") - defaultArguments["initial-cluster-state"] = "existing" + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "initial-cluster", strings.Join(endpoints, ","), 1) + defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "initial-cluster-state", "existing", 1) } command := []string{"etcd"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...) + command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.Etcd.Local.ExtraArgs)...) return command } diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 5fed54e5dc8..4aa181a51cc 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -176,7 +176,7 @@ func TestGetEtcdCommand(t *testing.T) { name string advertiseAddress string nodeName string - extraArgs map[string]string + extraArgs []kubeadmapi.Arg initialCluster []etcdutil.Member expected []string }{ @@ -243,9 +243,9 @@ func TestGetEtcdCommand(t *testing.T) { name: "Extra args", advertiseAddress: "1.2.3.4", nodeName: "bar", - extraArgs: map[string]string{ - "listen-client-urls": "https://10.0.1.10:2379", - "advertise-client-urls": "https://10.0.1.10:2379", + extraArgs: []kubeadmapi.Arg{ + {Name: "listen-client-urls", Value: "https://10.0.1.10:2379"}, + {Name: "advertise-client-urls", Value: "https://10.0.1.10:2379"}, }, expected: []string{ "etcd", diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 083fbfc0d2a..79e2dc564fb 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -51,7 +51,7 @@ func GetNodeNameAndHostname(cfg *kubeadmapi.NodeRegistrationOptions) (string, st if cfg.Name != "" { nodeName = cfg.Name } - if name, ok := cfg.KubeletExtraArgs["hostname-override"]; ok { + if name, idx := kubeadmapi.GetArgValue(cfg.KubeletExtraArgs, "hostname-override", -1); idx > -1 { nodeName = name } return nodeName, hostname, err @@ -65,23 +65,23 @@ func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *k pauseImage: images.GetPauseImage(cfg), registerTaintsUsingFlags: registerTaintsUsingFlags, } - stringMap := buildKubeletArgMap(flagOpts) - argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeReg.KubeletExtraArgs) + stringMap := buildKubeletArgs(flagOpts) + argList := kubeadmutil.ArgumentsToCommand(stringMap, nodeReg.KubeletExtraArgs) envFileContent := fmt.Sprintf("%s=%q\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir) } -// buildKubeletArgMapCommon takes a kubeletFlagsOpts object and builds based on that a string-string map with flags +// buildKubeletArgsCommon takes a kubeletFlagsOpts object and builds based on that a slice of arguments // that are common to both Linux and Windows -func buildKubeletArgMapCommon(opts kubeletFlagsOpts) map[string]string { - kubeletFlags := map[string]string{} - kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket +func buildKubeletArgsCommon(opts kubeletFlagsOpts) []kubeadmapi.Arg { + kubeletFlags := []kubeadmapi.Arg{} + kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "container-runtime-endpoint", Value: opts.nodeRegOpts.CRISocket}) // This flag passes the pod infra container image (e.g. "pause" image) to the kubelet // and prevents its garbage collection if opts.pauseImage != "" { - kubeletFlags["pod-infra-container-image"] = opts.pauseImage + kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "pod-infra-container-image", Value: opts.pauseImage}) } if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 { @@ -89,8 +89,7 @@ func buildKubeletArgMapCommon(opts kubeletFlagsOpts) map[string]string { for _, taint := range opts.nodeRegOpts.Taints { taintStrs = append(taintStrs, taint.ToString()) } - - kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",") + kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "register-with-taints", Value: strings.Join(taintStrs, ",")}) } // Pass the "--hostname-override" flag to the kubelet only if it's different from the hostname @@ -100,7 +99,7 @@ func buildKubeletArgMapCommon(opts kubeletFlagsOpts) map[string]string { } if nodeName != hostname { klog.V(1).Infof("setting kubelet hostname-override to %q", nodeName) - kubeletFlags["hostname-override"] = nodeName + kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "hostname-override", Value: nodeName}) } return kubeletFlags @@ -121,8 +120,8 @@ func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { return nil } -// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags +// buildKubeletArgs takes a kubeletFlagsOpts object and builds based on that a slice of arguments // that should be given to the local kubelet daemon. -func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { - return buildKubeletArgMapCommon(opts) +func buildKubeletArgs(opts kubeletFlagsOpts) []kubeadmapi.Arg { + return buildKubeletArgsCommon(opts) } diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index 6c0d8116a3a..b4881379574 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -27,23 +27,25 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) -func TestBuildKubeletArgMap(t *testing.T) { +func TestBuildKubeletArgs(t *testing.T) { tests := []struct { name string opts kubeletFlagsOpts - expected map[string]string + expected []kubeadmapi.Arg }{ { name: "hostname override", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "unix:///var/run/containerd/containerd.sock", - KubeletExtraArgs: map[string]string{"hostname-override": "override-name"}, + CRISocket: "unix:///var/run/containerd/containerd.sock", + KubeletExtraArgs: []kubeadmapi.Arg{ + {Name: "hostname-override", Value: "override-name"}, + }, }, }, - expected: map[string]string{ - "container-runtime-endpoint": "unix:///var/run/containerd/containerd.sock", - "hostname-override": "override-name", + expected: []kubeadmapi.Arg{ + {Name: "container-runtime-endpoint", Value: "unix:///var/run/containerd/containerd.sock"}, + {Name: "hostname-override", Value: "override-name"}, }, }, { @@ -66,9 +68,9 @@ func TestBuildKubeletArgMap(t *testing.T) { }, registerTaintsUsingFlags: true, }, - expected: map[string]string{ - "container-runtime-endpoint": "unix:///var/run/containerd/containerd.sock", - "register-with-taints": "foo=bar:baz,key=val:eff", + expected: []kubeadmapi.Arg{ + {Name: "container-runtime-endpoint", Value: "unix:///var/run/containerd/containerd.sock"}, + {Name: "register-with-taints", Value: "foo=bar:baz,key=val:eff"}, }, }, { @@ -79,19 +81,19 @@ func TestBuildKubeletArgMap(t *testing.T) { }, pauseImage: "registry.k8s.io/pause:3.9", }, - expected: map[string]string{ - "container-runtime-endpoint": "unix:///var/run/containerd/containerd.sock", - "pod-infra-container-image": "registry.k8s.io/pause:3.9", + expected: []kubeadmapi.Arg{ + {Name: "container-runtime-endpoint", Value: "unix:///var/run/containerd/containerd.sock"}, + {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:3.9"}, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - actual := buildKubeletArgMap(test.opts) + actual := buildKubeletArgs(test.opts) if !reflect.DeepEqual(actual, test.expected) { t.Errorf( - "failed buildKubeletArgMap:\n\texpected: %v\n\t actual: %v", + "failed buildKubeletArgs:\n\texpected: %v\n\t actual: %v", test.expected, actual, ) @@ -116,7 +118,9 @@ func TestGetNodeNameAndHostname(t *testing.T) { name: "overridden hostname", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - KubeletExtraArgs: map[string]string{"hostname-override": "override-name"}, + KubeletExtraArgs: []kubeadmapi.Arg{ + {Name: "hostname-override", Value: "override-name"}, + }, }, }, expectedNodeName: "override-name", @@ -126,7 +130,9 @@ func TestGetNodeNameAndHostname(t *testing.T) { name: "overridden hostname uppercase", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - KubeletExtraArgs: map[string]string{"hostname-override": "OVERRIDE-NAME"}, + KubeletExtraArgs: []kubeadmapi.Arg{ + {Name: "hostname-override", Value: "OVERRIDE-NAME"}, + }, }, }, expectedNodeName: "OVERRIDE-NAME", @@ -136,7 +142,9 @@ func TestGetNodeNameAndHostname(t *testing.T) { name: "hostname contains only spaces", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - KubeletExtraArgs: map[string]string{"hostname-override": " "}, + KubeletExtraArgs: []kubeadmapi.Arg{ + {Name: "hostname-override", Value: " "}, + }, }, }, expectedNodeName: " ", @@ -146,7 +154,9 @@ func TestGetNodeNameAndHostname(t *testing.T) { name: "empty parameter", opts: kubeletFlagsOpts{ nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - KubeletExtraArgs: map[string]string{"hostname-override": ""}, + KubeletExtraArgs: []kubeadmapi.Arg{ + {Name: "hostname-override", Value: ""}, + }, }, }, expectedNodeName: "", diff --git a/cmd/kubeadm/app/util/arguments.go b/cmd/kubeadm/app/util/arguments.go index 72c79d9be2a..82ab5d48a50 100644 --- a/cmd/kubeadm/app/util/arguments.go +++ b/cmd/kubeadm/app/util/arguments.go @@ -24,41 +24,54 @@ import ( "github.com/pkg/errors" "k8s.io/klog/v2" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) -// BuildArgumentListFromMap takes two string-string maps, one with the base arguments and one +// ArgumentsToCommand takes two Arg slices, one with the base arguments and one // with optional override arguments. In the return list override arguments will precede base -// arguments -func BuildArgumentListFromMap(baseArguments map[string]string, overrideArguments map[string]string) []string { +// arguments. If an argument is present in the overrides, it will cause +// all instances of the same argument in the base list to be discarded, leaving +// only the instances of this argument in the overrides to be applied. +func ArgumentsToCommand(base []kubeadmapi.Arg, overrides []kubeadmapi.Arg) []string { var command []string - var keys []string + // Copy the base arguments into a new slice. + args := make([]kubeadmapi.Arg, len(base)) + copy(args, base) - argsMap := make(map[string]string) - - for k, v := range baseArguments { - argsMap[k] = v + // Go trough the override arguments and delete all instances of arguments with the same name + // in the base list of arguments. + for i := 0; i < len(overrides); i++ { + repeat: + for j := 0; j < len(args); j++ { + if overrides[i].Name == args[j].Name { + // Remove this existing argument and search for another argument + // with the same name in base. + args = append(args[:j], args[j+1:]...) + goto repeat + } + } } - for k, v := range overrideArguments { - argsMap[k] = v - } + // Concatenate the overrides and the base arguments and then sort them. + args = append(args, overrides...) + sort.Slice(args, func(i, j int) bool { + if args[i].Name == args[j].Name { + return args[i].Value < args[j].Value + } + return args[i].Name < args[j].Name + }) - for k := range argsMap { - keys = append(keys, k) - } - - sort.Strings(keys) - for _, k := range keys { - command = append(command, fmt.Sprintf("--%s=%s", k, argsMap[k])) + for _, arg := range args { + command = append(command, fmt.Sprintf("--%s=%s", arg.Name, arg.Value)) } return command } -// ParseArgumentListToMap parses a CLI argument list in the form "--foo=bar" to a string-string map -func ParseArgumentListToMap(arguments []string) map[string]string { - resultingMap := map[string]string{} - for i, arg := range arguments { +// ArgumentsFromCommand parses a CLI command in the form "--foo=bar" to an Arg slice +func ArgumentsFromCommand(command []string) []kubeadmapi.Arg { + args := []kubeadmapi.Arg{} + for i, arg := range command { key, val, err := parseArgument(arg) // Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name @@ -70,24 +83,16 @@ func ParseArgumentListToMap(arguments []string) map[string]string { continue } - resultingMap[key] = val + args = append(args, kubeadmapi.Arg{Name: key, Value: val}) } - return resultingMap -} -// ReplaceArgument gets a command list; converts it to a map for easier modification, runs the provided function that -// returns a new modified map, and then converts the map back to a command string slice -func ReplaceArgument(command []string, argMutateFunc func(map[string]string) map[string]string) []string { - argMap := ParseArgumentListToMap(command) - - // Save the first command (the executable) if we're sure it's not an argument (i.e. no --) - var newCommand []string - if len(command) > 0 && !strings.HasPrefix(command[0], "--") { - newCommand = append(newCommand, command[0]) - } - newArgMap := argMutateFunc(argMap) - newCommand = append(newCommand, BuildArgumentListFromMap(newArgMap, map[string]string{})...) - return newCommand + sort.Slice(args, func(i, j int) bool { + if args[i].Name == args[j].Name { + return args[i].Value < args[j].Value + } + return args[i].Name < args[j].Name + }) + return args } // parseArgument parses the argument "--foo=bar" to "foo" and "bar" diff --git a/cmd/kubeadm/app/util/arguments_test.go b/cmd/kubeadm/app/util/arguments_test.go index 31bc9e88a3a..a796293fb1d 100644 --- a/cmd/kubeadm/app/util/arguments_test.go +++ b/cmd/kubeadm/app/util/arguments_test.go @@ -20,23 +20,25 @@ import ( "reflect" "sort" "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) -func TestBuildArgumentListFromMap(t *testing.T) { +func TestArgumentsToCommand(t *testing.T) { var tests = []struct { name string - base map[string]string - overrides map[string]string + base []kubeadmapi.Arg + overrides []kubeadmapi.Arg expected []string }{ { name: "override an argument from the base", - base: map[string]string{ - "admission-control": "NamespaceLifecycle", - "allow-privileged": "true", + base: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle"}, + {Name: "allow-privileged", Value: "true"}, }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", + overrides: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, }, expected: []string{ "--admission-control=NamespaceLifecycle,LimitRanger", @@ -44,12 +46,43 @@ func TestBuildArgumentListFromMap(t *testing.T) { }, }, { - name: "add an argument that is not in base", - base: map[string]string{ - "allow-privileged": "true", + name: "override an argument from the base and add duplicate", + base: []kubeadmapi.Arg{ + {Name: "token-auth-file", Value: "/token"}, + {Name: "tls-sni-cert-key", Value: "/some/path/"}, }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", + overrides: []kubeadmapi.Arg{ + {Name: "tls-sni-cert-key", Value: "/some/new/path"}, + {Name: "tls-sni-cert-key", Value: "/some/new/path/subpath"}, + }, + expected: []string{ + "--tls-sni-cert-key=/some/new/path", + "--tls-sni-cert-key=/some/new/path/subpath", + "--token-auth-file=/token", + }, + }, + { + name: "override all duplicate arguments from base", + base: []kubeadmapi.Arg{ + {Name: "token-auth-file", Value: "/token"}, + {Name: "tls-sni-cert-key", Value: "foo"}, + {Name: "tls-sni-cert-key", Value: "bar"}, + }, + overrides: []kubeadmapi.Arg{ + {Name: "tls-sni-cert-key", Value: "/some/new/path"}, + }, + expected: []string{ + "--tls-sni-cert-key=/some/new/path", + "--token-auth-file=/token", + }, + }, + { + name: "add an argument that is not in base", + base: []kubeadmapi.Arg{ + {Name: "allow-privileged", Value: "true"}, + }, + overrides: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, }, expected: []string{ "--admission-control=NamespaceLifecycle,LimitRanger", @@ -58,12 +91,12 @@ func TestBuildArgumentListFromMap(t *testing.T) { }, { name: "allow empty strings in base", - base: map[string]string{ - "allow-privileged": "true", - "something-that-allows-empty-string": "", + base: []kubeadmapi.Arg{ + {Name: "allow-privileged", Value: "true"}, + {Name: "something-that-allows-empty-string", Value: ""}, }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", + overrides: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, }, expected: []string{ "--admission-control=NamespaceLifecycle,LimitRanger", @@ -73,13 +106,13 @@ func TestBuildArgumentListFromMap(t *testing.T) { }, { name: "allow empty strings in overrides", - base: map[string]string{ - "allow-privileged": "true", - "something-that-allows-empty-string": "foo", + base: []kubeadmapi.Arg{ + {Name: "allow-privileged", Value: "true"}, + {Name: "something-that-allows-empty-string", Value: "foo"}, }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "something-that-allows-empty-string": "", + overrides: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, + {Name: "something-that-allows-empty-string", Value: ""}, }, expected: []string{ "--admission-control=NamespaceLifecycle,LimitRanger", @@ -91,19 +124,19 @@ func TestBuildArgumentListFromMap(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { - actual := BuildArgumentListFromMap(rt.base, rt.overrides) + actual := ArgumentsToCommand(rt.base, rt.overrides) if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed BuildArgumentListFromMap:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) + t.Errorf("failed ArgumentsToCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) } }) } } -func TestParseArgumentListToMap(t *testing.T) { +func TestArgumentsFromCommand(t *testing.T) { var tests = []struct { - name string - args []string - expectedMap map[string]string + name string + args []string + expected []kubeadmapi.Arg }{ { name: "normal case", @@ -111,9 +144,9 @@ func TestParseArgumentListToMap(t *testing.T) { "--admission-control=NamespaceLifecycle,LimitRanger", "--allow-privileged=true", }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "allow-privileged": "true", + expected: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, + {Name: "allow-privileged", Value: "true"}, }, }, { @@ -123,10 +156,10 @@ func TestParseArgumentListToMap(t *testing.T) { "--allow-privileged=true", "--feature-gates=EnableFoo=true,EnableBar=false", }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "allow-privileged": "true", - "feature-gates": "EnableFoo=true,EnableBar=false", + expected: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, + {Name: "allow-privileged", Value: "true"}, + {Name: "feature-gates", Value: "EnableFoo=true,EnableBar=false"}, }, }, { @@ -137,75 +170,32 @@ func TestParseArgumentListToMap(t *testing.T) { "--allow-privileged=true", "--feature-gates=EnableFoo=true,EnableBar=false", }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "allow-privileged": "true", - "feature-gates": "EnableFoo=true,EnableBar=false", + expected: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, + {Name: "allow-privileged", Value: "true"}, + {Name: "feature-gates", Value: "EnableFoo=true,EnableBar=false"}, + }, + }, + { + name: "allow duplicate args", + args: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--tls-sni-cert-key=/some/path", + "--tls-sni-cert-key=/some/path/subpath", + }, + expected: []kubeadmapi.Arg{ + {Name: "admission-control", Value: "NamespaceLifecycle,LimitRanger"}, + {Name: "tls-sni-cert-key", Value: "/some/path"}, + {Name: "tls-sni-cert-key", Value: "/some/path/subpath"}, }, }, } for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { - actualMap := ParseArgumentListToMap(rt.args) - if !reflect.DeepEqual(actualMap, rt.expectedMap) { - t.Errorf("failed ParseArgumentListToMap:\nexpected:\n%v\nsaw:\n%v", rt.expectedMap, actualMap) - } - }) - } -} - -func TestReplaceArgument(t *testing.T) { - var tests = []struct { - name string - args []string - mutateFunc func(map[string]string) map[string]string - expectedArgs []string - }{ - { - name: "normal case", - args: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - }, - mutateFunc: func(argMap map[string]string) map[string]string { - argMap["admission-control"] = "NamespaceLifecycle,LimitRanger,ResourceQuota" - return argMap - }, - expectedArgs: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger,ResourceQuota", - "--allow-privileged=true", - }, - }, - { - name: "another normal case", - args: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - }, - mutateFunc: func(argMap map[string]string) map[string]string { - argMap["new-arg-here"] = "foo" - return argMap - }, - expectedArgs: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - "--new-arg-here=foo", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualArgs := ReplaceArgument(rt.args, rt.mutateFunc) - sort.Strings(actualArgs) - sort.Strings(rt.expectedArgs) - if !reflect.DeepEqual(actualArgs, rt.expectedArgs) { - t.Errorf("failed ReplaceArgument:\nexpected:\n%v\nsaw:\n%v", rt.expectedArgs, actualArgs) + actual := ArgumentsFromCommand(rt.args) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed ArgumentsFromCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) } }) } @@ -236,7 +226,7 @@ func TestRoundtrip(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { // These two methods should be each other's opposite functions, test that by chaining the methods and see if you get the same result back - actual := BuildArgumentListFromMap(ParseArgumentListToMap(rt.args), map[string]string{}) + actual := ArgumentsToCommand(ArgumentsFromCommand(rt.args), []kubeadmapi.Arg{}) sort.Strings(actual) sort.Strings(rt.args) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index cab4feb16bf..eae22677477 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -295,7 +295,7 @@ func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { // GetControllerManagerProbeAddress returns the kubernetes controller manager probe address func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists { + if addr, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, kubeControllerManagerBindAddressArg, -1); idx > -1 { return getProbeAddress(addr) } return "127.0.0.1" @@ -303,7 +303,7 @@ func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) stri // GetSchedulerProbeAddress returns the kubernetes scheduler probe address func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists { + if addr, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, kubeSchedulerBindAddressArg, -1); idx > -1 { return getProbeAddress(addr) } return "127.0.0.1" @@ -320,7 +320,7 @@ func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int32, v1. if cfg.Local == nil || cfg.Local.ExtraArgs == nil { return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP } - if arg, exists := cfg.Local.ExtraArgs["listen-metrics-urls"]; exists { + if arg, idx := kubeadmapi.GetArgValue(cfg.Local.ExtraArgs, "listen-metrics-urls", -1); idx > -1 { // Use the first url in the listen-metrics-urls if multiple URL's are specified. arg = strings.Split(arg, ",")[0] parsedURL, err := url.Parse(arg) diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index bcea14876d1..1dd6863e6eb 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -107,7 +107,7 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { desc: "no controller manager extra args leads to 127.0.0.1 being used", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{}, + ExtraArgs: []kubeadmapi.Arg{}, }, }, expected: "127.0.0.1", @@ -116,8 +116,8 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { desc: "setting controller manager extra address arg to something acknowledges it", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "10.10.10.10", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeControllerManagerBindAddressArg, Value: "10.10.10.10"}, }, }, }, @@ -127,8 +127,8 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { desc: "setting controller manager extra ipv6 address arg to something acknowledges it", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "2001:abcd:bcda::1", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeControllerManagerBindAddressArg, Value: "2001:abcd:bcda::1"}, }, }, }, @@ -138,8 +138,8 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { desc: "setting controller manager extra address arg to 0.0.0.0 returns empty", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "0.0.0.0", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeControllerManagerBindAddressArg, Value: "0.0.0.0"}, }, }, }, @@ -149,8 +149,8 @@ func TestGetControllerManagerProbeAddress(t *testing.T) { desc: "setting controller manager extra ipv6 address arg to :: returns empty", cfg: &kubeadmapi.ClusterConfiguration{ ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "::", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeControllerManagerBindAddressArg, Value: "::"}, }, }, }, @@ -178,7 +178,7 @@ func TestGetSchedulerProbeAddress(t *testing.T) { desc: "no scheduler extra args leads to 127.0.0.1 being used", cfg: &kubeadmapi.ClusterConfiguration{ Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{}, + ExtraArgs: []kubeadmapi.Arg{}, }, }, expected: "127.0.0.1", @@ -187,8 +187,8 @@ func TestGetSchedulerProbeAddress(t *testing.T) { desc: "setting scheduler extra address arg to something acknowledges it", cfg: &kubeadmapi.ClusterConfiguration{ Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "10.10.10.10", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeSchedulerBindAddressArg, Value: "10.10.10.10"}, }, }, }, @@ -198,8 +198,8 @@ func TestGetSchedulerProbeAddress(t *testing.T) { desc: "setting scheduler extra ipv6 address arg to something acknowledges it", cfg: &kubeadmapi.ClusterConfiguration{ Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "2001:abcd:bcda::1", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeSchedulerBindAddressArg, Value: "2001:abcd:bcda::1"}, }, }, }, @@ -209,8 +209,8 @@ func TestGetSchedulerProbeAddress(t *testing.T) { desc: "setting scheduler extra ipv6 address arg to 0.0.0.0 returns empty", cfg: &kubeadmapi.ClusterConfiguration{ Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "0.0.0.0", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeSchedulerBindAddressArg, Value: "0.0.0.0"}, }, }, }, @@ -220,8 +220,8 @@ func TestGetSchedulerProbeAddress(t *testing.T) { desc: "setting scheduler extra ipv6 address arg to :: returns empty", cfg: &kubeadmapi.ClusterConfiguration{ Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "::", + ExtraArgs: []kubeadmapi.Arg{ + {Name: kubeSchedulerBindAddressArg, Value: "::"}, }, }, }, @@ -251,8 +251,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe URL from two URLs", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://1.2.3.4:1234,https://4.3.2.1:2381"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "https://1.2.3.4:1234,https://4.3.2.1:2381"}, + }, }, }, isIPv6: false, @@ -264,8 +265,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe URL with HTTP scheme", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://1.2.3.4:1234"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "http://1.2.3.4:1234"}, + }, }, }, isIPv6: false, @@ -277,8 +279,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe URL without scheme should result in defaults", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "1.2.3.4"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "1.2.3.4"}, + }, }, }, isIPv6: false, @@ -290,8 +293,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe URL without port", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://1.2.3.4"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "https://1.2.3.4"}, + }, }, }, isIPv6: false, @@ -303,8 +307,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe URL from two IPv6 URLs", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://[2001:abcd:bcda::1]:1234,https://[2001:abcd:bcda::2]:2381"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "https://[2001:abcd:bcda::1]:1234,https://[2001:abcd:bcda::2]:2381"}, + }, }, }, isIPv6: true, @@ -316,8 +321,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe localhost IPv6 URL with HTTP scheme", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://[::1]:1234"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "http://[::1]:1234"}, + }, }, }, isIPv6: true, @@ -329,8 +335,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe IPv6 URL with HTTP scheme", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://[2001:abcd:bcda::1]:1234"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "http://[2001:abcd:bcda::1]:1234"}, + }, }, }, isIPv6: true, @@ -342,8 +349,9 @@ func TestGetEtcdProbeEndpoint(t *testing.T) { name: "etcd probe IPv6 URL without port", cfg: &kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://[2001:abcd:bcda::1]"}, + ExtraArgs: []kubeadmapi.Arg{ + {Name: "listen-metrics-urls", Value: "https://[2001:abcd:bcda::1]"}, + }, }, }, isIPv6: true,