Merge pull request #77012 from rosti/certkey-v1beta2

kubeadm: Add certificateKey field to v1beta2 config

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -35,6 +35,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 		fuzzLocalEtcd,
 		fuzzNetworking,
 		fuzzJoinConfiguration,
+		fuzzJoinControlPlane,
 	}
 }
 
@@ -81,6 +82,9 @@ func fuzzInitConfiguration(obj *kubeadm.InitConfiguration, c fuzz.Continue) {
 			Usages: []string{"foo"},
 		},
 	}
+
+	// Pin values for fields that are not present in v1beta1
+	obj.CertificateKey = ""
 }
 
 func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue) {
@@ -135,3 +139,10 @@ func fuzzJoinConfiguration(obj *kubeadm.JoinConfiguration, c fuzz.Continue) {
 		Timeout:           &metav1.Duration{Duration: 1234},
 	}
 }
+
+func fuzzJoinControlPlane(obj *kubeadm.JoinControlPlane, c fuzz.Continue) {
+	c.FuzzNoCustom(obj)
+
+	// Pin values for fields that are not present in v1beta1
+	obj.CertificateKey = ""
+}

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -50,6 +50,10 @@ type InitConfiguration struct {
 	// on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process
 	// fails you may set the desired value here.
 	LocalAPIEndpoint APIEndpoint
+
+	// CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in
+	// a secret in the cluster during the uploadcerts init phase.
+	CertificateKey string
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -332,6 +336,10 @@ type JoinConfiguration struct {
 type JoinControlPlane struct {
 	// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
 	LocalAPIEndpoint APIEndpoint
+
+	// CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret
+	// upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration.
+	CertificateKey string
 }
 
 // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

cmd/kubeadm/app/apis/kubeadm/v1beta1/BUILD

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "bootstraptokenstring.go",
+        "conversion.go",
         "defaults.go",
         "defaults_unix.go",
         "defaults_windows.go",
@@ -32,9 +33,15 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["bootstraptokenstring_test.go"],
+    srcs = [
+        "bootstraptokenstring_test.go",
+        "conversion_test.go",
+    ],
     embed = [":go_default_library"],
-    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
 )
 
 filegroup(

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+func Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s); err != nil {
+		return err
+	}
+
+	if in.CertificateKey != "" {
+		return errors.New("certificateKey field is not supported by v1beta1 config format")
+	}
+
+	return nil
+}
+
+func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s); err != nil {
+		return err
+	}
+
+	if in.CertificateKey != "" {
+		return errors.New("certificateKey field is not supported by v1beta1 config format")
+	}
+
+	return nil
+}

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+func TestInternalToVersionedInitConfigurationConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.InitConfiguration
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.InitConfiguration{},
+			expectedError: false,
+		},
+		"certificateKey set causes an error": {
+			in: kubeadm.InitConfiguration{
+				CertificateKey: "secret",
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &InitConfiguration{}
+			err := Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestInternalToVersionedJoinControlPlaneConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.JoinControlPlane
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.JoinControlPlane{},
+			expectedError: false,
+		},
+		"certificateKey set causes an error": {
+			in: kubeadm.JoinControlPlane{
+				CertificateKey: "secret",
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &JoinControlPlane{}
+			err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go

@@ -247,6 +247,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddConversionFunc((*kubeadm.JoinControlPlane)(nil), (*JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(a.(*kubeadm.JoinControlPlane), b.(*JoinControlPlane), scope)
+	}); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -690,14 +700,10 @@ func autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kube
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	// WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type
 	return nil
 }
 
-// Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration is an autogenerated conversion function.
-func Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error {
-	return autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s)
-}
-
 func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error {
 	if err := Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
 		return err
@@ -706,7 +712,15 @@ func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *Join
 	if err := Convert_v1beta1_Discovery_To_kubeadm_Discovery(&in.Discovery, &out.Discovery, s); err != nil {
 		return err
 	}
-	out.ControlPlane = (*kubeadm.JoinControlPlane)(unsafe.Pointer(in.ControlPlane))
+	if in.ControlPlane != nil {
+		in, out := &in.ControlPlane, &out.ControlPlane
+		*out = new(kubeadm.JoinControlPlane)
+		if err := Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ControlPlane = nil
+	}
 	return nil
 }
 
@@ -723,7 +737,15 @@ func autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kube
 	if err := Convert_kubeadm_Discovery_To_v1beta1_Discovery(&in.Discovery, &out.Discovery, s); err != nil {
 		return err
 	}
-	out.ControlPlane = (*JoinControlPlane)(unsafe.Pointer(in.ControlPlane))
+	if in.ControlPlane != nil {
+		in, out := &in.ControlPlane, &out.ControlPlane
+		*out = new(JoinControlPlane)
+		if err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ControlPlane = nil
+	}
 	return nil
 }
 
@@ -748,14 +770,10 @@ func autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubead
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	// WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type
 	return nil
 }
 
-// Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane is an autogenerated conversion function.
-func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error {
-	return autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s)
-}
-
 func autoConvert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error {
 	if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil {
 		return err

cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go

@@ -52,6 +52,10 @@ type InitConfiguration struct {
 	// on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process
 	// fails you may set the desired value here.
 	LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"`
+
+	// CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in
+	// a secret in the cluster during the uploadcerts init phase.
+	CertificateKey string `json:"certificateKey,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -321,6 +325,10 @@ type JoinConfiguration struct {
 type JoinControlPlane struct {
 	// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
 	LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"`
+
+	// CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret
+	// upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration.
+	CertificateKey string `json:"certificateKey,omitempty"`
 }
 
 // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go

@@ -671,6 +671,7 @@ func autoConvert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(in *Init
 	if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -690,6 +691,7 @@ func autoConvert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(in *kube
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -736,6 +738,7 @@ func autoConvert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinCo
 	if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 
@@ -748,6 +751,7 @@ func autoConvert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(in *kubead
 	if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil {
 		return err
 	}
+	out.CertificateKey = in.CertificateKey
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -50,6 +50,7 @@ func ValidateInitConfiguration(c *kubeadm.InitConfiguration) field.ErrorList {
 	allErrs = append(allErrs, ValidateBootstrapTokens(c.BootstrapTokens, field.NewPath("bootstrapTokens"))...)
 	allErrs = append(allErrs, ValidateClusterConfiguration(&c.ClusterConfiguration)...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, field.NewPath("localAPIEndpoint"))...)
+	// TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key
 	return allErrs
 }
 
@@ -91,6 +92,7 @@ func ValidateJoinControlPlane(c *kubeadm.JoinControlPlane, fldPath *field.Path)
 	allErrs := field.ErrorList{}
 	if c != nil {
 		allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, fldPath.Child("localAPIEndpoint"))...)
+		// TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key
 	}
 	return allErrs
 }

cmd/kubeadm/app/cmd/init.go

@@ -97,7 +97,6 @@ type initOptions struct {
 	bto                     *options.BootstrapTokenOptions
 	externalcfg             *kubeadmapiv1beta2.InitConfiguration
 	uploadCerts             bool
-	certificateKey          string
 	skipCertificateKeyPrint bool
 }
 
@@ -119,7 +118,6 @@ type initData struct {
 	client                  clientset.Interface
 	outputWriter            io.Writer
 	uploadCerts             bool
-	certificateKey          string
 	skipCertificateKeyPrint bool
 }
 
@@ -231,6 +229,10 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.InitConfig
 		&cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name,
 		`Specify the node name.`,
 	)
+	flagSet.StringVar(
+		&cfg.CertificateKey, options.CertificateKey, "",
+		"Key used to encrypt the control-plane certificates in the kubeadm-certs Secret.",
+	)
 	cmdutil.AddCRISocketFlag(flagSet, &cfg.NodeRegistration.CRISocket)
 	options.AddFeatureGatesStringFlag(flagSet, featureGatesString)
 }
@@ -255,10 +257,6 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) {
 		&initOptions.uploadCerts, options.UploadCerts, initOptions.uploadCerts,
 		"Upload control-plane certificates to the kubeadm-certs Secret.",
 	)
-	flagSet.StringVar(
-		&initOptions.certificateKey, options.CertificateKey, "",
-		"Key used to encrypt the control-plane certificates in the kubeadm-certs Secret.",
-	)
 	flagSet.BoolVar(
 		&initOptions.skipCertificateKeyPrint, options.SkipCertificateKeyPrint, initOptions.skipCertificateKeyPrint,
 		"Don't print the key used to encrypt the control-plane certificates.",
@@ -387,7 +385,6 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		externalCA:              externalCA,
 		outputWriter:            out,
 		uploadCerts:             options.uploadCerts,
-		certificateKey:          options.certificateKey,
 		skipCertificateKeyPrint: options.skipCertificateKeyPrint,
 	}, nil
 }
@@ -399,12 +396,12 @@ func (d *initData) UploadCerts() bool {
 
 // CertificateKey returns the key used to encrypt the certs.
 func (d *initData) CertificateKey() string {
-	return d.certificateKey
+	return d.cfg.CertificateKey
 }
 
 // SetCertificateKey set the key used to encrypt the certs.
 func (d *initData) SetCertificateKey(key string) {
-	d.certificateKey = key
+	d.cfg.CertificateKey = key
 }
 
 // SkipCertificateKeyPrint returns the skipCertificateKeyPrint flag.
@@ -518,7 +515,7 @@ func (d *initData) Tokens() []string {
 }
 
 func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error {
-	joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.certificateKey, i.skipTokenPrint, i.skipCertificateKeyPrint)
+	joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.CertificateKey(), i.skipTokenPrint, i.skipCertificateKeyPrint)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/cmd/join.go

@@ -128,7 +128,6 @@ type joinOptions struct {
 	controlPlane          bool
 	ignorePreflightErrors []string
 	externalcfg           *kubeadmapiv1beta2.JoinConfiguration
-	certificateKey        string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -143,7 +142,6 @@ type joinData struct {
 	clientSet             *clientset.Clientset
 	ignorePreflightErrors sets.String
 	outputWriter          io.Writer
-	certificateKey        string
 }
 
 // NewCmdJoin returns "kubeadm join" command.
@@ -221,6 +219,10 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.JoinConfig
 		&cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name,
 		`Specify the node name.`,
 	)
+	flagSet.StringVar(
+		&cfg.ControlPlane.CertificateKey, options.CertificateKey, "",
+		"Use this key to decrypt the certificate secrets uploaded by init.",
+	)
 	// add control plane endpoint flags to the specified flagset
 	flagSet.StringVar(
 		&cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress,
@@ -273,10 +275,6 @@ func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
 		&joinOptions.controlPlane, options.ControlPlane, joinOptions.controlPlane,
 		"Create a new control plane instance on this node",
 	)
-	flagSet.StringVar(
-		&joinOptions.certificateKey, options.CertificateKey, "",
-		"Use this key to decrypt the certificate secrets uploaded by init.",
-	)
 }
 
 // newJoinOptions returns a struct ready for being used for creating cmd join flags.
@@ -404,13 +402,15 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 		tlsBootstrapCfg:       tlsBootstrapCfg,
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		outputWriter:          out,
-		certificateKey:        opt.certificateKey,
 	}, nil
 }
 
 // CertificateKey returns the key used to encrypt the certs.
 func (j *joinData) CertificateKey() string {
-	return j.certificateKey
+	if j.cfg.ControlPlane != nil {
+		return j.cfg.ControlPlane.CertificateKey
+	}
+	return ""
 }
 
 // Cfg returns the JoinConfiguration.

cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml

@@ -1,5 +1,6 @@
 CACertPath: /etc/kubernetes/pki/ca.crt
 ControlPlane:
+  CertificateKey: ""
   LocalAPIEndpoint:
     AdvertiseAddress: 192.168.2.2
     BindPort: 6443