diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index a06ad8f6a33..b488fda5f65 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -85,6 +85,10 @@ func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue obj.APIServer.TimeoutForControlPlane = &metav1.Duration{ Duration: constants.DefaultControlPlaneTimeout, } + obj.ControllerManager.ExtraEnvs = []corev1.EnvVar{} + obj.APIServer.ExtraEnvs = []corev1.EnvVar{} + obj.Scheduler.ExtraEnvs = []corev1.EnvVar{} + obj.Etcd.Local.ExtraEnvs = []corev1.EnvVar{} } func fuzzDNS(obj *kubeadm.DNS, c fuzz.Continue) { diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 35a0ea64b63..e4c471179b8 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -150,6 +150,11 @@ type ControlPlaneComponent struct { // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. ExtraVolumes []HostPathMount + + // ExtraEnvs is an extra set of environment variables to pass to the control plane component. + // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. + // +optional + ExtraEnvs []v1.EnvVar } // APIServer holds settings necessary for API server deployments in the cluster @@ -262,6 +267,11 @@ type LocalEtcd struct { // command line except without leading dash(es). ExtraArgs map[string]string + // ExtraEnvs is an extra set of environment variables to pass to the control plane component. + // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. + // +optional + ExtraEnvs []v1.EnvVar + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. ServerCertSANs []string // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go index 155ae224879..b77842990c1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta3 import ( + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/conversion" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -34,3 +35,23 @@ func Convert_v1beta3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConf err = Convert_v1beta3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(&ClusterConfiguration{}, &out.ClusterConfiguration, s) return err } + +// Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent is required due to the missing ControlPlaneComponent.ExtraEnvs in v1beta3. +func Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { + out.ExtraEnvs = []v1.EnvVar{} + return autoConvert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in, out, s) +} + +func Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { + return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in, out, s) +} + +// Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd is required due to the missing LocalEtcd.ExtraEnvs in v1beta3. +func Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { + out.ExtraEnvs = []v1.EnvVar{} + return autoConvert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) +} + +func Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { + return autoConvert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in, out, s) +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go index f6edf530cf7..e9f6c0cf6fa 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/zz_generated.conversion.go @@ -79,16 +79,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*ControlPlaneComponent)(nil), (*kubeadm.ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(a.(*ControlPlaneComponent), b.(*kubeadm.ControlPlaneComponent), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ControlPlaneComponent)(nil), (*ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(a.(*kubeadm.ControlPlaneComponent), b.(*ControlPlaneComponent), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*DNS)(nil), (*kubeadm.DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_DNS_To_kubeadm_DNS(a.(*DNS), b.(*kubeadm.DNS), scope) }); err != nil { @@ -179,16 +169,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.LocalEtcd)(nil), (*LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(a.(*kubeadm.LocalEtcd), b.(*LocalEtcd), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Networking)(nil), (*kubeadm.Networking)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_Networking_To_kubeadm_Networking(a.(*Networking), b.(*kubeadm.Networking), scope) }); err != nil { @@ -219,16 +199,36 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.ControlPlaneComponent)(nil), (*ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(a.(*kubeadm.ControlPlaneComponent), b.(*ControlPlaneComponent), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_kubeadm_InitConfiguration_To_v1beta3_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.LocalEtcd)(nil), (*LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(a.(*kubeadm.LocalEtcd), b.(*LocalEtcd), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*ControlPlaneComponent)(nil), (*kubeadm.ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(a.(*ControlPlaneComponent), b.(*kubeadm.ControlPlaneComponent), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*InitConfiguration)(nil), (*kubeadm.InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_InitConfiguration_To_kubeadm_InitConfiguration(a.(*InitConfiguration), b.(*kubeadm.InitConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) + }); err != nil { + return err + } return nil } @@ -383,22 +383,13 @@ func autoConvert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent( return nil } -// Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent is an autogenerated conversion function. -func Convert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_v1beta3_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in, out, s) -} - func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) + // WARNING: in.ExtraEnvs requires manual conversion: does not exist in peer-type return nil } -// Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent is an autogenerated conversion function. -func Convert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta3_ControlPlaneComponent(in, out, s) -} - func autoConvert_v1beta3_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { if err := Convert_v1beta3_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { return err @@ -450,7 +441,15 @@ func Convert_kubeadm_Discovery_To_v1beta3_Discovery(in *kubeadm.Discovery, out * } func autoConvert_v1beta3_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - out.Local = (*kubeadm.LocalEtcd)(unsafe.Pointer(in.Local)) + if in.Local != nil { + in, out := &in.Local, &out.Local + *out = new(kubeadm.LocalEtcd) + if err := Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(*in, *out, s); err != nil { + return err + } + } else { + out.Local = nil + } out.External = (*kubeadm.ExternalEtcd)(unsafe.Pointer(in.External)) return nil } @@ -461,7 +460,15 @@ func Convert_v1beta3_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s convers } func autoConvert_kubeadm_Etcd_To_v1beta3_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - out.Local = (*LocalEtcd)(unsafe.Pointer(in.Local)) + if in.Local != nil { + in, out := &in.Local, &out.Local + *out = new(LocalEtcd) + if err := Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(*in, *out, s); err != nil { + return err + } + } else { + out.Local = nil + } out.External = (*ExternalEtcd)(unsafe.Pointer(in.External)) return nil } @@ -671,27 +678,18 @@ func autoConvert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kube return nil } -// Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd is an autogenerated conversion function. -func Convert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - return autoConvert_v1beta3_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) -} - func autoConvert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { if err := Convert_kubeadm_ImageMeta_To_v1beta3_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { return err } out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + // WARNING: in.ExtraEnvs requires manual conversion: does not exist in peer-type out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil } -// Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd is an autogenerated conversion function. -func Convert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_LocalEtcd_To_v1beta3_LocalEtcd(in, out, s) -} - func autoConvert_v1beta3_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { out.ServiceSubnet = in.ServiceSubnet out.PodSubnet = in.PodSubnet diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go index 5056e492e7f..aa02f911e1b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/doc.go @@ -20,10 +20,13 @@ limitations under the License. // +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm // Package v1beta4 defines the v1beta4 version of the kubeadm configuration file format. -// This version improves on the v1beta2 format by fixing some minor issues and adding a few new fields. +// This version improves on the v1beta3 format by fixing some minor issues and adding a few new fields. // // A list of changes since v1beta3: +// // - TODO https://github.com/kubernetes/kubeadm/issues/2890 +// - Support custom environment variables in control plane components under `ClusterConfiguration`. +// Use `APIServer.ExtraEnvs`, `ControllerManager.ExtraEnvs`, `Scheduler.ExtraEnvs`, `Etcd.Local.ExtraEnvs`. // // Migration from old kubeadm config versions // @@ -39,7 +42,7 @@ limitations under the License. // configuration options defined in the kubeadm config file are also available as command line flags, but only // the most common/simple use case are supported with this approach. // -// A kubeadm config file could contain multiple configuration types separated using three dashes (“---”). +// A kubeadm config file could contain multiple configuration types separated using three dashes ("---"). // // kubeadm supports the following configuration types: // diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go index 5d07b270c6e..217839456ce 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go @@ -150,6 +150,11 @@ type ControlPlaneComponent struct { // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. // +optional ExtraVolumes []HostPathMount `json:"extraVolumes,omitempty"` + + // ExtraEnvs is an extra set of environment variables to pass to the control plane component. + // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. + // +optional + ExtraEnvs []corev1.EnvVar `json:"extraEnvs,omitempty"` } // APIServer holds settings necessary for API server deployments in the cluster @@ -282,6 +287,11 @@ type LocalEtcd struct { // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` + // ExtraEnvs is an extra set of environment variables to pass to the control plane component. + // Environment variables passed using ExtraEnvs will override any existing environment variables, or *_proxy environment variables that kubeadm adds by default. + // +optional + ExtraEnvs []corev1.EnvVar `json:"extraEnvs,omitempty"` + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. // +optional ServerCertSANs []string `json:"serverCertSANs,omitempty"` diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go index 3126823e2ef..b1bc05f7614 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go @@ -380,6 +380,7 @@ func Convert_kubeadm_ClusterConfiguration_To_v1beta4_ClusterConfiguration(in *ku func autoConvert_v1beta4_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) + out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) return nil } @@ -391,6 +392,7 @@ func Convert_v1beta4_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in * func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta4_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) + out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) return nil } @@ -666,6 +668,7 @@ func autoConvert_v1beta4_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kube } out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil @@ -682,6 +685,7 @@ func autoConvert_kubeadm_LocalEtcd_To_v1beta4_LocalEtcd(in *kubeadm.LocalEtcd, o } out.DataDir = in.DataDir out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.ExtraEnvs = *(*[]corev1.EnvVar)(unsafe.Pointer(&in.ExtraEnvs)) out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) return nil diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go index af9f47743a6..9966096aa56 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go @@ -145,6 +145,13 @@ func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { *out = make([]HostPathMount, len(*in)) copy(*out, *in) } + if in.ExtraEnvs != nil { + in, out := &in.ExtraEnvs, &out.ExtraEnvs + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -415,6 +422,13 @@ func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { (*out)[key] = val } } + if in.ExtraEnvs != nil { + in, out := &in.ExtraEnvs, &out.ExtraEnvs + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.ServerCertSANs != nil { in, out := &in.ServerCertSANs, &out.ServerCertSANs *out = make([]string, len(*in)) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.defaults.go index c898c74a884..cd12c914e2b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.defaults.go @@ -23,6 +23,7 @@ package v1beta4 import ( runtime "k8s.io/apimachinery/pkg/runtime" + v1 "k8s.io/kubernetes/pkg/apis/core/v1" ) // RegisterDefaults adds defaulters functions to the given scheme. @@ -37,7 +38,41 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_ClusterConfiguration(in *ClusterConfiguration) { SetDefaults_ClusterConfiguration(in) + if in.Etcd.Local != nil { + for i := range in.Etcd.Local.ExtraEnvs { + a := &in.Etcd.Local.ExtraEnvs[i] + if a.ValueFrom != nil { + if a.ValueFrom.FieldRef != nil { + v1.SetDefaults_ObjectFieldSelector(a.ValueFrom.FieldRef) + } + } + } + } SetDefaults_APIServer(&in.APIServer) + for i := range in.APIServer.ControlPlaneComponent.ExtraEnvs { + a := &in.APIServer.ControlPlaneComponent.ExtraEnvs[i] + if a.ValueFrom != nil { + if a.ValueFrom.FieldRef != nil { + v1.SetDefaults_ObjectFieldSelector(a.ValueFrom.FieldRef) + } + } + } + for i := range in.ControllerManager.ExtraEnvs { + a := &in.ControllerManager.ExtraEnvs[i] + if a.ValueFrom != nil { + if a.ValueFrom.FieldRef != nil { + v1.SetDefaults_ObjectFieldSelector(a.ValueFrom.FieldRef) + } + } + } + for i := range in.Scheduler.ExtraEnvs { + a := &in.Scheduler.ExtraEnvs[i] + if a.ValueFrom != nil { + if a.ValueFrom.FieldRef != nil { + v1.SetDefaults_ObjectFieldSelector(a.ValueFrom.FieldRef) + } + } + } } func SetObjectDefaults_InitConfiguration(in *InitConfiguration) { diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 6c4b65757a9..5aa117d5435 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -174,6 +174,13 @@ func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { *out = make([]HostPathMount, len(*in)) copy(*out, *in) } + if in.ExtraEnvs != nil { + in, out := &in.ExtraEnvs, &out.ExtraEnvs + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -445,6 +452,13 @@ func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { (*out)[key] = val } } + if in.ExtraEnvs != nil { + in, out := &in.ExtraEnvs, &out.ExtraEnvs + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.ServerCertSANs != nil { in, out := &in.ServerCertSANs, &out.ServerCertSANs *out = make([]string, len(*in)) diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go index 8d0ebbc3703..6516284985c 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff.go @@ -148,7 +148,7 @@ func runDiff(flags *diffFlags, args []string) error { cfg.ClusterConfiguration.KubernetesVersion = flags.newK8sVersionStr - specs := controlplane.GetStaticPodSpecs(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint) + specs := controlplane.GetStaticPodSpecs(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, nil) for spec, pod := range specs { var path string switch spec { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 73f4fa56270..1c57110171f 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -48,9 +48,12 @@ func CreateInitStaticPodManifestFiles(manifestDir, patchesDir string, cfg *kubea // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration // NB. this method holds the information about how kubeadm creates static pod manifests. -func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) map[string]v1.Pod { +func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, proxyEnvs []v1.EnvVar) map[string]v1.Pod { // Get the required hostpath mounts mounts := getHostPathVolumesForTheControlPlane(cfg) + if proxyEnvs == nil { + proxyEnvs = kubeadmutil.GetProxyEnvVars() + } // Prepare static pod specs staticPodSpecs := map[string]v1.Pod{ @@ -64,7 +67,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), Resources: staticpodutil.ComponentResources("250m"), - Env: kubeadmutil.GetProxyEnvVars(), + Env: kubeadmutil.MergeEnv(proxyEnvs, cfg.APIServer.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ @@ -76,7 +79,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), Resources: staticpodutil.ComponentResources("200m"), - Env: kubeadmutil.GetProxyEnvVars(), + Env: kubeadmutil.MergeEnv(proxyEnvs, cfg.ControllerManager.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeScheduler, @@ -87,7 +90,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), Resources: staticpodutil.ComponentResources("100m"), - Env: kubeadmutil.GetProxyEnvVars(), + Env: kubeadmutil.MergeEnv(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } return staticPodSpecs @@ -97,7 +100,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, componentNames ...string) error { // gets the StaticPodSpecs, actualized for the current ClusterConfiguration klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") - specs := GetStaticPodSpecs(cfg, endpoint) + specs := GetStaticPodSpecs(cfg, endpoint, nil) var usersAndGroups *users.UsersAndGroups var err error diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index e5af8566dab..5ff2c1db285 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -30,6 +30,7 @@ import ( "github.com/lithammer/dedent" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -51,14 +52,18 @@ func TestGetStaticPodSpecs(t *testing.T) { // Creates a Cluster Configuration cfg := &kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.9.0", + Scheduler: kubeadmapi.ControlPlaneComponent{ExtraEnvs: []v1.EnvVar{ + {Name: "Foo", Value: "Bar"}, + }}, } // Executes GetStaticPodSpecs - specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{}) + specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{}, []v1.EnvVar{}) var tests = []struct { name string staticPodName string + env []v1.EnvVar }{ { name: "KubeAPIServer", @@ -71,6 +76,7 @@ func TestGetStaticPodSpecs(t *testing.T) { { name: "KubeScheduler", staticPodName: kubeadmconstants.KubeScheduler, + env: []v1.EnvVar{{Name: "Foo", Value: "Bar"}}, }, } @@ -78,12 +84,15 @@ func TestGetStaticPodSpecs(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // assert the spec for the staticPodName exists if spec, ok := specs[tc.staticPodName]; ok { - // Assert each specs refers to the right pod if spec.Spec.Containers[0].Name != tc.staticPodName { t.Errorf("getKubeConfigSpecs spec for %s contains pod %s, expects %s", tc.staticPodName, spec.Spec.Containers[0].Name, tc.staticPodName) } - + if tc.env != nil { + if !reflect.DeepEqual(spec.Spec.Containers[0].Env, tc.env) { + t.Errorf("expected env: %v, got: %v", tc.env, spec.Spec.Containers[0].Env) + } + } } else { t.Errorf("getStaticPodSpecs didn't create spec for %s ", tc.staticPodName) } diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index daa8cc8b67a..129db183cc9 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -224,6 +224,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A }, LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health?exclude=NOSPACE&serializable=true", probePort, probeScheme), StartupProbe: staticpodutil.StartupProbe(probeHostname, "/health?serializable=false", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane), + Env: cfg.Etcd.Local.ExtraEnvs, }, etcdMounts, // etcd will listen on the advertise address of the API server, in a different port (2379) diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index 12658d5de41..5fed54e5dc8 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -29,6 +29,7 @@ import ( "github.com/lithammer/dedent" + v1 "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" @@ -43,6 +44,9 @@ func TestGetEtcdPodSpec(t *testing.T) { Etcd: kubeadmapi.Etcd{ Local: &kubeadmapi.LocalEtcd{ DataDir: "/var/lib/etcd", + ExtraEnvs: []v1.EnvVar{ + {Name: "Foo", Value: "Bar"}, + }, }, }, } @@ -55,6 +59,10 @@ func TestGetEtcdPodSpec(t *testing.T) { if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd { t.Errorf("getKubeConfigSpecs spec for etcd contains pod %s, expects %s", spec.Spec.Containers[0].Name, kubeadmconstants.Etcd) } + env := []v1.EnvVar{{Name: "Foo", Value: "Bar"}} + if !reflect.DeepEqual(spec.Spec.Containers[0].Env, env) { + t.Errorf("expected env: %v, got: %v", env, spec.Spec.Containers[0].Env) + } } func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { diff --git a/cmd/kubeadm/app/util/env.go b/cmd/kubeadm/app/util/env.go index c4196272493..ea8de76d2eb 100644 --- a/cmd/kubeadm/app/util/env.go +++ b/cmd/kubeadm/app/util/env.go @@ -41,3 +41,18 @@ func GetProxyEnvVars() []v1.EnvVar { } return envs } + +// MergeEnv merges values of environment variable slices. The values defined in later slices overwrite values in previous ones. +func MergeEnv(envList ...[]v1.EnvVar) []v1.EnvVar { + m := make(map[string]v1.EnvVar) + merged := []v1.EnvVar{} + for _, envs := range envList { + for _, env := range envs { + m[env.Name] = env + } + } + for _, v := range m { + merged = append(merged, v) + } + return merged +} diff --git a/cmd/kubeadm/app/util/env_test.go b/cmd/kubeadm/app/util/env_test.go new file mode 100644 index 00000000000..e1bc6a5aad6 --- /dev/null +++ b/cmd/kubeadm/app/util/env_test.go @@ -0,0 +1,76 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + v1 "k8s.io/api/core/v1" +) + +func TestMergeEnv(t *testing.T) { + baseEnv := []v1.EnvVar{} + extraEnv := []v1.EnvVar{} + MergeEnv(append(baseEnv, extraEnv...)) + var tests = []struct { + name string + proxyEnv []v1.EnvVar + extraEnv []v1.EnvVar + mergedEnv []v1.EnvVar + }{ + { + name: "normal case without duplicated env", + proxyEnv: []v1.EnvVar{ + {Name: "Foo1", Value: "Bar1"}, + {Name: "Foo2", Value: "Bar2"}, + }, + extraEnv: []v1.EnvVar{ + {Name: "Foo3", Value: "Bar3"}, + }, + mergedEnv: []v1.EnvVar{ + {Name: "Foo1", Value: "Bar1"}, + {Name: "Foo2", Value: "Bar2"}, + {Name: "Foo3", Value: "Bar3"}, + }, + }, + { + name: "extraEnv env take precedence over the proxyEnv", + proxyEnv: []v1.EnvVar{ + {Name: "Foo1", Value: "Bar1"}, + {Name: "Foo2", Value: "Bar2"}, + }, + extraEnv: []v1.EnvVar{ + {Name: "Foo2", Value: "Bar3"}, + }, + mergedEnv: []v1.EnvVar{ + {Name: "Foo1", Value: "Bar1"}, + {Name: "Foo2", Value: "Bar3"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + envs := MergeEnv(test.proxyEnv, test.extraEnv) + if !assert.ElementsMatch(t, envs, test.mergedEnv) { + t.Errorf("expected env: %v, got: %v", test.mergedEnv, envs) + } + }) + } +}