kubeadm: Add certificateKey field to v1beta2 config

This change introduces config fields to the v1beta2 format, that allow
certificate key to be specified in the config file. This certificate key is a
hex encoded AES key, that is used to encrypt certificates and keys, needed for
secondary control plane nodes to join. The same key is used for the decryption
during control plane join.
It is important to note, that this key is never uploaded to the cluster. It can
only be specified on either command line or the config file.
The new fields can be used like so:

---
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
certificateKey: "yourSecretHere"
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: JoinConfiguration
controlPlane:
  certificateKey: "yourSecretHere"
---

Signed-off-by: Rostislav M. Georgiev <rostislavg@vmware.com>

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -35,6 +35,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 		fuzzLocalEtcd,
 		fuzzNetworking,
 		fuzzJoinConfiguration,
+		fuzzJoinControlPlane,
 	}
 }
 
@@ -81,6 +82,9 @@ func fuzzInitConfiguration(obj *kubeadm.InitConfiguration, c fuzz.Continue) {
 			Usages: []string{"foo"},
 		},
 	}
+
+	// Pin values for fields that are not present in v1beta1
+	obj.CertificateKey = ""
 }
 
 func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue) {
@@ -135,3 +139,10 @@ func fuzzJoinConfiguration(obj *kubeadm.JoinConfiguration, c fuzz.Continue) {
 		Timeout:           &metav1.Duration{Duration: 1234},
 	}
 }
+
+func fuzzJoinControlPlane(obj *kubeadm.JoinControlPlane, c fuzz.Continue) {
+	c.FuzzNoCustom(obj)
+
+	// Pin values for fields that are not present in v1beta1
+	obj.CertificateKey = ""
+}

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -50,6 +50,10 @@ type InitConfiguration struct {
 	// on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process
 	// fails you may set the desired value here.
 	LocalAPIEndpoint APIEndpoint
+
+	// CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in
+	// a secret in the cluster during the uploadcerts init phase.
+	CertificateKey string
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -332,6 +336,10 @@ type JoinConfiguration struct {
 type JoinControlPlane struct {
 	// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
 	LocalAPIEndpoint APIEndpoint
+
+	// CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret
+	// upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration.
+	CertificateKey string
 }
 
 // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

cmd/kubeadm/app/apis/kubeadm/v1beta1/BUILD

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "bootstraptokenstring.go",
+        "conversion.go",
         "defaults.go",
         "defaults_unix.go",
         "defaults_windows.go",
@@ -32,9 +33,15 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["bootstraptokenstring_test.go"],
+    srcs = [
+        "bootstraptokenstring_test.go",
+        "conversion_test.go",
+    ],
     embed = [":go_default_library"],
-    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
 )
 
 filegroup(

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+func Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s); err != nil {
+		return err
+	}
+
+	if in.CertificateKey != "" {
+		return errors.New("certificateKey field is not supported by v1beta1 config format")
+	}
+
+	return nil
+}
+
+func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s); err != nil {
+		return err
+	}
+
+	if in.CertificateKey != "" {
+		return errors.New("certificateKey field is not supported by v1beta1 config format")
+	}
+
+	return nil
+}

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+func TestInternalToVersionedInitConfigurationConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.InitConfiguration
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.InitConfiguration{},
+			expectedError: false,
+		},
+		"certificateKey set causes an error": {
+			in: kubeadm.InitConfiguration{
+				CertificateKey: "secret",
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &InitConfiguration{}
+			err := Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestInternalToVersionedJoinControlPlaneConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.JoinControlPlane
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.JoinControlPlane{},
+			expectedError: false,
+		},
+		"certificateKey set causes an error": {
+			in: kubeadm.JoinControlPlane{
+				CertificateKey: "secret",
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &JoinControlPlane{}
+			err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go

@@ -247,6 +247,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddConversionFunc((*kubeadm.JoinControlPlane)(nil), (*JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(a.(*kubeadm.JoinControlPlane), b.(*JoinControlPlane), scope)
+	}); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -690,14 +700,10 @@ func autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kube
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	// WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type
 	return nil
 }
 
-// Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration is an autogenerated conversion function.
-func Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error {
-	return autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s)
-}
-
 func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error {
 	if err := Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
 		return err
@@ -706,7 +712,15 @@ func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *Join
 	if err := Convert_v1beta1_Discovery_To_kubeadm_Discovery(&in.Discovery, &out.Discovery, s); err != nil {
 		return err
 	}
-	out.ControlPlane = (*kubeadm.JoinControlPlane)(unsafe.Pointer(in.ControlPlane))
+	if in.ControlPlane != nil {
+		in, out := &in.ControlPlane, &out.ControlPlane
+		*out = new(kubeadm.JoinControlPlane)
+		if err := Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ControlPlane = nil
+	}
 	return nil
 }
 
@@ -723,7 +737,15 @@ func autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kube
 	if err := Convert_kubeadm_Discovery_To_v1beta1_Discovery(&in.Discovery, &out.Discovery, s); err != nil {
 		return err
 	}
-	out.ControlPlane = (*JoinControlPlane)(unsafe.Pointer(in.ControlPlane))
+	if in.ControlPlane != nil {
+		in, out := &in.ControlPlane, &out.ControlPlane
+		*out = new(JoinControlPlane)
+		if err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ControlPlane = nil
+	}
 	return nil
 }
 
@@ -748,14 +770,10 @@ func autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubead
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	// WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type
 	return nil
 }
 
-// Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane is an autogenerated conversion function.
-func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error {
-	return autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s)
-}
-
 func autoConvert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error {
 	if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil {
 		return err

cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go

@@ -52,6 +52,10 @@ type InitConfiguration struct {
 	// on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process
 	// fails you may set the desired value here.
 	LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"`
+
+	// CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in
+	// a secret in the cluster during the uploadcerts init phase.
+	CertificateKey string `json:"certificateKey,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -321,6 +325,10 @@ type JoinConfiguration struct {
 type JoinControlPlane struct {
 	// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
 	LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"`
+
+	// CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret
+	// upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration.
+	CertificateKey string `json:"certificateKey,omitempty"`
 }
 
 // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go

@@ -671,6 +671,7 @@ func autoConvert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(in *Init
 	if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -690,6 +691,7 @@ func autoConvert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(in *kube
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -736,6 +738,7 @@ func autoConvert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinCo
 	if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -748,6 +751,7 @@ func autoConvert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(in *kubead
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -50,6 +50,7 @@ func ValidateInitConfiguration(c *kubeadm.InitConfiguration) field.ErrorList {
 	allErrs = append(allErrs, ValidateBootstrapTokens(c.BootstrapTokens, field.NewPath("bootstrapTokens"))...)
 	allErrs = append(allErrs, ValidateClusterConfiguration(&c.ClusterConfiguration)...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, field.NewPath("localAPIEndpoint"))...)
+	// TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key
 	return allErrs
 }
 
@@ -91,6 +92,7 @@ func ValidateJoinControlPlane(c *kubeadm.JoinControlPlane, fldPath *field.Path)
 	allErrs := field.ErrorList{}
 	if c != nil {
 		allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, fldPath.Child("localAPIEndpoint"))...)
+		// TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key
 	}
 	return allErrs
 }

cmd/kubeadm/app/cmd/init.go

@@ -97,7 +97,6 @@ type initOptions struct {
 	bto                     *options.BootstrapTokenOptions
 	externalcfg             *kubeadmapiv1beta2.InitConfiguration
 	uploadCerts             bool
-	certificateKey          string
 	skipCertificateKeyPrint bool
 }
 
@@ -120,7 +119,6 @@ type initData struct {
 	waiter                  apiclient.Waiter
 	outputWriter            io.Writer
 	uploadCerts             bool
-	certificateKey          string
 	skipCertificateKeyPrint bool
 }
 
@@ -232,6 +230,10 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.InitConfig
 		&cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name,
 		`Specify the node name.`,
 	)
+	flagSet.StringVar(
+		&cfg.CertificateKey, options.CertificateKey, "",
+		"Key used to encrypt the control-plane certificates in the kubeadm-certs Secret.",
+	)
 	cmdutil.AddCRISocketFlag(flagSet, &cfg.NodeRegistration.CRISocket)
 	options.AddFeatureGatesStringFlag(flagSet, featureGatesString)
 }
@@ -256,10 +258,6 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) {
 		&initOptions.uploadCerts, options.UploadCerts, initOptions.uploadCerts,
 		"Upload control-plane certificates to the kubeadm-certs Secret.",
 	)
-	flagSet.StringVar(
-		&initOptions.certificateKey, options.CertificateKey, "",
-		"Key used to encrypt the control-plane certificates in the kubeadm-certs Secret.",
-	)
 	flagSet.BoolVar(
 		&initOptions.skipCertificateKeyPrint, options.SkipCertificateKeyPrint, initOptions.skipCertificateKeyPrint,
 		"Don't print the key used to encrypt the control-plane certificates.",
@@ -388,7 +386,6 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		externalCA:              externalCA,
 		outputWriter:            out,
 		uploadCerts:             options.uploadCerts,
-		certificateKey:          options.certificateKey,
 		skipCertificateKeyPrint: options.skipCertificateKeyPrint,
 	}, nil
 }
@@ -400,12 +397,12 @@ func (d *initData) UploadCerts() bool {
 
 // CertificateKey returns the key used to encrypt the certs.
 func (d *initData) CertificateKey() string {
-	return d.certificateKey
+	return d.cfg.CertificateKey
 }
 
 // SetCertificateKey set the key used to encrypt the certs.
 func (d *initData) SetCertificateKey(key string) {
-	d.certificateKey = key
+	d.cfg.CertificateKey = key
 }
 
 // SkipCertificateKeyPrint returns the skipCertificateKeyPrint flag.
@@ -519,7 +516,7 @@ func (d *initData) Tokens() []string {
 }
 
 func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error {
-	joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.certificateKey, i.skipTokenPrint, i.skipCertificateKeyPrint)
+	joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.CertificateKey(), i.skipTokenPrint, i.skipCertificateKeyPrint)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/cmd/join.go

@@ -128,7 +128,6 @@ type joinOptions struct {
 	controlPlane          bool
 	ignorePreflightErrors []string
 	externalcfg           *kubeadmapiv1beta2.JoinConfiguration
-	certificateKey        string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -144,7 +143,6 @@ type joinData struct {
 	clientSet             *clientset.Clientset
 	ignorePreflightErrors sets.String
 	outputWriter          io.Writer
-	certificateKey        string
 }
 
 // NewCmdJoin returns "kubeadm join" command.
@@ -222,6 +220,10 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.JoinConfig
 		&cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name,
 		`Specify the node name.`,
 	)
+	flagSet.StringVar(
+		&cfg.ControlPlane.CertificateKey, options.CertificateKey, "",
+		"Use this key to decrypt the certificate secrets uploaded by init.",
+	)
 	// add control plane endpoint flags to the specified flagset
 	flagSet.StringVar(
 		&cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress,
@@ -274,10 +276,6 @@ func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
 		&joinOptions.controlPlane, options.ControlPlane, joinOptions.controlPlane,
 		"Create a new control plane instance on this node",
 	)
-	flagSet.StringVar(
-		&joinOptions.certificateKey, options.CertificateKey, "",
-		"Use this key to decrypt the certificate secrets uploaded by init.",
-	)
 }
 
 // newJoinOptions returns a struct ready for being used for creating cmd join flags.
@@ -405,13 +403,15 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 		tlsBootstrapCfg:       tlsBootstrapCfg,
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		outputWriter:          out,
-		certificateKey:        opt.certificateKey,
 	}, nil
 }
 
 // CertificateKey returns the key used to encrypt the certs.
 func (j *joinData) CertificateKey() string {
-	return j.certificateKey
+	if j.cfg.ControlPlane != nil {
+		return j.cfg.ControlPlane.CertificateKey
+	}
+	return ""
 }
 
 // Cfg returns the JoinConfiguration.

cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml

@@ -1,5 +1,6 @@
 CACertPath: /etc/kubernetes/pki/ca.crt
 ControlPlane:
+  CertificateKey: ""
   LocalAPIEndpoint:
     AdvertiseAddress: 192.168.2.2
     BindPort: 6443