diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index 288378c6563..eb41c1bf540 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -48,7 +48,6 @@ filegroup( "//cmd/kubeadm/app/phases/kubelet:all-srcs", "//cmd/kubeadm/app/phases/markcontrolplane:all-srcs", "//cmd/kubeadm/app/phases/patchnode:all-srcs", - "//cmd/kubeadm/app/phases/selfhosting:all-srcs", "//cmd/kubeadm/app/phases/upgrade:all-srcs", "//cmd/kubeadm/app/phases/uploadconfig:all-srcs", "//cmd/kubeadm/app/preflight:all-srcs", diff --git a/cmd/kubeadm/app/cmd/alpha/BUILD b/cmd/kubeadm/app/cmd/alpha/BUILD index 6aa1659f536..ad6e92597b7 100644 --- a/cmd/kubeadm/app/cmd/alpha/BUILD +++ b/cmd/kubeadm/app/cmd/alpha/BUILD @@ -6,7 +6,6 @@ go_library( "alpha.go", "certs.go", "kubeconfig.go", - "selfhosting.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha", visibility = ["//visibility:public"], @@ -14,18 +13,13 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", "//cmd/kubeadm/app/phases/copycerts:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/alpha.go b/cmd/kubeadm/app/cmd/alpha/alpha.go index b3e68bd9f54..95085d8a854 100644 --- a/cmd/kubeadm/app/cmd/alpha/alpha.go +++ b/cmd/kubeadm/app/cmd/alpha/alpha.go @@ -33,12 +33,6 @@ func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command { deprecateCommand(`please use the same command under "kubeadm kubeconfig"`, kubeconfigCmd) cmd.AddCommand(kubeconfigCmd) - const shDeprecatedMessage = "self-hosting support in kubeadm is deprecated " + - "and will be removed in a future release" - shCommand := newCmdSelfhosting(in) - deprecateCommand(shDeprecatedMessage, shCommand) - cmd.AddCommand(shCommand) - certsCommand := NewCmdCertsUtility(out) deprecateCommand(`please use the same command under "kubeadm certs"`, certsCommand) cmd.AddCommand(certsCommand) diff --git a/cmd/kubeadm/app/cmd/alpha/selfhosting.go b/cmd/kubeadm/app/cmd/alpha/selfhosting.go deleted file mode 100644 index 5617b6abc0a..00000000000 --- a/cmd/kubeadm/app/cmd/alpha/selfhosting.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "bufio" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - selfhostingLongDesc = cmdutil.LongDesc(` - Convert static Pod files for control plane components into self-hosted DaemonSets configured via the Kubernetes API. - - See the documentation for self-hosting limitations. - - ` + cmdutil.AlphaDisclaimer) - - selfhostingExample = cmdutil.Examples(` - # Convert a static Pod-hosted control plane into a self-hosted one. - - kubeadm alpha phase self-hosting convert-from-staticpods - `) -) - -// newCmdSelfhosting returns the self-hosting Cobra command -func newCmdSelfhosting(in io.Reader) *cobra.Command { - cmd := &cobra.Command{ - Use: "selfhosting", - Aliases: []string{"selfhosted", "self-hosting"}, - Short: "Make a kubeadm cluster self-hosted", - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(getSelfhostingSubCommand(in)) - return cmd -} - -// getSelfhostingSubCommand returns sub commands for Self-hosting phase -func getSelfhostingSubCommand(in io.Reader) *cobra.Command { - - cfg := &kubeadmapiv1beta2.ClusterConfiguration{} - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath, featureGatesString, kubeConfigFile string - forcePivot, certsInSecrets := false, false - - // Creates the UX Command - cmd := &cobra.Command{ - Use: "pivot", - Aliases: []string{"from-staticpods"}, - Short: "Convert a static Pod-hosted control plane into a self-hosted one", - Long: selfhostingLongDesc, - Example: selfhostingExample, - RunE: func(cmd *cobra.Command, args []string) error { - - var err error - - if !forcePivot { - fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!") - fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ") - s := bufio.NewScanner(in) - s.Scan() - - if err = s.Err(); err != nil { - return err - } - - if strings.ToLower(s.Text()) != "y" { - return errors.New("aborted pivot operation") - } - } - - fmt.Println("[pivot] pivoting cluster to self-hosted") - - if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { - return err - } - - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - return err - } - - // Gets the Kubernetes client - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - if err != nil { - return err - } - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing LoadOrDefaultInitConfiguration - phases.SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, cfg) - if err != nil { - return err - } - - // Converts the Static Pod-hosted control plane into a self-hosted one - waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - return selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets) - }, - Args: cobra.NoArgs, - } - - // Add flags to the command - // flags bound to the configuration object - cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) - options.AddConfigFlag(cmd.Flags(), &cfgPath) - - cmd.Flags().BoolVarP( - &certsInSecrets, "store-certs-in-secrets", "s", - false, "Enable storing certs in secrets") - - cmd.Flags().BoolVarP( - &forcePivot, "force", "f", false, - "Pivot the cluster without prompting for confirmation", - ) - - // flags that are not bound to the configuration object - // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - - return cmd -} diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 71b2fab6bcc..87f138558c8 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -127,11 +127,6 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } - // Check if the cluster is self-hosted - if upgrade.IsControlPlaneSelfHosted(client) { - return nil, nil, nil, errors.New("cannot upgrade a self-hosted control plane") - } - // Fetch the configuration from a file or ConfigMap and validate it fmt.Println("[upgrade/config] Making sure the configuration is correct:") diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 213bcbde6c2..8336b2e21ee 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -304,9 +304,6 @@ const ( // Kubelet defines variable used internally when referring to the Kubelet Kubelet = "kubelet" - // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has - SelfHostingPrefix = "self-hosted-" - // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) KubeCertificatesVolumeName = "k8s-certs" @@ -555,11 +552,6 @@ func GetKubeletKubeConfigPath() string { return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) } -// AddSelfHostedPrefix adds the self-hosted- prefix to the component name -func AddSelfHostedPrefix(componentName string) string { - return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName) -} - // CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { tempDir := path.Join(KubernetesDir, TempDirForKubeadm) diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index 83bc8814d63..d4a33b2d631 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -110,41 +110,6 @@ func TestGetStaticPodFilepath(t *testing.T) { } } -func TestAddSelfHostedPrefix(t *testing.T) { - var tests = []struct { - componentName, expected string - }{ - { - componentName: "kube-apiserver", - expected: "self-hosted-kube-apiserver", - }, - { - componentName: "kube-controller-manager", - expected: "self-hosted-kube-controller-manager", - }, - { - componentName: "kube-scheduler", - expected: "self-hosted-kube-scheduler", - }, - { - componentName: "foo", - expected: "self-hosted-foo", - }, - } - for _, rt := range tests { - t.Run(rt.componentName, func(t *testing.T) { - actual := AddSelfHostedPrefix(rt.componentName) - if actual != rt.expected { - t.Errorf( - "failed AddSelfHostedPrefix:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - func TestEtcdSupportedVersion(t *testing.T) { var supportedEtcdVersion = map[uint8]string{ 13: "3.2.24", diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD deleted file mode 100644 index c5eba468d2e..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = [ - "podspec_mutation_test.go", - "selfhosting_test.go", - "selfhosting_volumes_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "podspec_mutation.go", - "selfhosting.go", - "selfhosting_volumes.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go deleted file mode 100644 index 0d4090798c6..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "path/filepath" - "strings" - - v1 "k8s.io/api/core/v1" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - // selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted - // Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from - // the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki) - selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig" -) - -// PodSpecMutatorFunc is a function capable of mutating a PodSpec -type PodSpecMutatorFunc func(*v1.PodSpec) - -// GetDefaultMutators gets the mutator functions that always should be used -func GetDefaultMutators() map[string][]PodSpecMutatorFunc { - return map[string][]PodSpecMutatorFunc{ - kubeadmconstants.KubeAPIServer: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - setHostIPOnPodSpec, - }, - kubeadmconstants.KubeControllerManager: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - }, - kubeadmconstants.KubeScheduler: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - }, - } -} - -// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed -func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc { - // Here the map of different mutators to use for the control plane's podspec is stored - mutators := GetDefaultMutators() - - if certsInSecrets { - // Some extra work to be done if we should store the control plane certificates in Secrets - // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them - mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) - mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) - mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) - } - - return mutators -} - -// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting -func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) { - // Get the mutator functions for the component in question, then loop through and execute them - mutatorsForComponent := mutators[name] - for _, mutateFunc := range mutatorsForComponent { - mutateFunc(podSpec) - } -} - -// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the control-plane label -func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) { - if podSpec.NodeSelector == nil { - podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleOldControlPlane: ""} - return - } - - podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleOldControlPlane] = "" -} - -// setControlPlaneTolerationOnPodSpec makes the Pod tolerate the control-plane taint -func setControlPlaneTolerationOnPodSpec(podSpec *v1.PodSpec) { - if podSpec.Tolerations == nil { - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - podSpec.Tolerations = []v1.Toleration{kubeadmconstants.OldControlPlaneToleration} - return - } - - podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.OldControlPlaneToleration) -} - -// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API -func setHostIPOnPodSpec(podSpec *v1.PodSpec) { - envVar := v1.EnvVar{ - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - } - - podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar) - - for i := range podSpec.Containers[0].Command { - if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") { - podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)" - } - } -} - -// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary -func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet -} - -// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeCertificatesVolumeName { - podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource() - } - } -} - -// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeCertificatesVolumeName { - podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource() - } else if v.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName) - } - } - - // Change directory for the kubeconfig directory to selfHostedKubeConfigDir - for i, vm := range podSpec.Containers[0].VolumeMounts { - if vm.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir - } - } - - // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) - // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes - // don't support that. - podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { - controllerManagerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName) - argMap["kubeconfig"] = controllerManagerKubeConfigPath - if _, ok := argMap["authentication-kubeconfig"]; ok { - argMap["authentication-kubeconfig"] = controllerManagerKubeConfigPath - } - if _, ok := argMap["authorization-kubeconfig"]; ok { - argMap["authorization-kubeconfig"] = controllerManagerKubeConfigPath - } - return argMap - }) -} - -// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName) - } - } - - // Change directory for the kubeconfig directory to selfHostedKubeConfigDir - for i, vm := range podSpec.Containers[0].VolumeMounts { - if vm.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir - } - } - - // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) - // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes - // don't support that. - podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { - schedulerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName) - argMap["kubeconfig"] = schedulerKubeConfigPath - if _, ok := argMap["authentication-kubeconfig"]; ok { - argMap["authentication-kubeconfig"] = schedulerKubeConfigPath - } - if _, ok := argMap["authorization-kubeconfig"]; ok { - argMap["authorization-kubeconfig"] = schedulerKubeConfigPath - } - return argMap - }) -} diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go deleted file mode 100644 index 81e69b9e225..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ /dev/null @@ -1,590 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "reflect" - "sort" - "testing" - - v1 "k8s.io/api/core/v1" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestMutatePodSpec(t *testing.T) { - var tests = []struct { - name string - component string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "mutate api server podspec", - component: kubeadmconstants.KubeAPIServer, - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=10.0.0.1", - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=$(HOST_IP)", - }, - Env: []v1.EnvVar{ - { - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }, - }, - }, - }, - - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "mutate controller manager podspec", - component: kubeadmconstants.KubeControllerManager, - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "mutate scheduler podspec", - component: kubeadmconstants.KubeScheduler, - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestAddNodeSelectorToPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - }, - }, - { - name: "podspec with a valid node selector", - podSpec: &v1.PodSpec{ - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - "foo": "bar", - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - addNodeSelectorToPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetControlPlaneTolerationOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - }, - }, - { - name: "podspec with a valid toleration", - podSpec: &v1.PodSpec{ - Tolerations: []v1.Toleration{ - {Key: "foo", Value: "bar"}, - }, - }, - expected: v1.PodSpec{ - Tolerations: []v1.Toleration{ - {Key: "foo", Value: "bar"}, - kubeadmconstants.OldControlPlaneToleration, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setControlPlaneTolerationOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setControlPlaneTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetRightDNSPolicyOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "podspec with a v1.DNSClusterFirst policy", - podSpec: &v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirst, - }, - expected: v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setRightDNSPolicyOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetHostIPOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set HOST_IP env var on a podspec", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=10.0.0.1", - }, - Env: []v1.EnvVar{}, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=$(HOST_IP)", - }, - Env: []v1.EnvVar{ - { - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setHostIPOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setHostIPOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForAPIServer(t *testing.T) { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for api server", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - }, - Command: []string{ - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/pki", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - }, - Command: []string{ - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: apiServerCertificatesVolumeSource(), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForAPIServer(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForAPIServer:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForControllerManager(t *testing.T) { - hostPathFileOrCreate := v1.HostPathFileOrCreate - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for controller mananger", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/controller-manager.conf", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/controller-manager.conf", - "--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf", - "--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/pki", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/controller-manager.conf", - Type: &hostPathFileOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/kubeconfig", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: controllerManagerCertificatesVolumeSource(), - }, - { - Name: "kubeconfig", - VolumeSource: kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForControllerManager(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForControllerManager:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForScheduler(t *testing.T) { - hostPathFileOrCreate := v1.HostPathFileOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for scheduler", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/scheduler.conf", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/scheduler.conf", - "--authentication-kubeconfig=/etc/kubernetes/scheduler.conf", - "--authorization-kubeconfig=/etc/kubernetes/scheduler.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/scheduler.conf", - Type: &hostPathFileOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/kubeconfig", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--authentication-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--authorization-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "kubeconfig", - VolumeSource: kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForScheduler(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForScheduler:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go deleted file mode 100644 index 1d3ad4d4f09..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "fmt" - "io/ioutil" - "os" - "time" - - "k8s.io/klog/v2" - - "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientset "k8s.io/client-go/kubernetes" - clientscheme "k8s.io/client-go/kubernetes/scheme" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out - selfHostingWaitTimeout = 2 * time.Minute - - // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets - selfHostingFailureThreshold int = 5 -) - -// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one -// It achieves that task this way: -// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests) -// 2. Extract the PodSpec from that Static Pod specification -// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly) -// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec -// 5. Create the DaemonSet resource. Wait until the Pods are running. -// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running. -// 7. The self-hosted containers should now step up and take over. -// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue -// Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes -// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error { - klog.V(1).Infoln("creating self hosted control plane") - // Adjust the timeout slightly to something self-hosting specific - waiter.SetTimeout(selfHostingWaitTimeout) - - // Here the map of different mutators to use for the control plane's PodSpec is stored - klog.V(1).Infoln("getting mutators") - mutators := GetMutatorsFromFeatureGates(certsInSecrets) - - if certsInSecrets { - // Upload the certificates and kubeconfig files from disk to the cluster as Secrets - if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { - return err - } - if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil { - return err - } - } - - for _, componentName := range kubeadmconstants.ControlPlaneComponents { - start := time.Now() - manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir) - - // Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist - if _, err := os.Stat(manifestPath); err != nil { - fmt.Printf("[self-hosted] The Static Pod for the component %q doesn't seem to be on the disk; trying the next one\n", componentName) - continue - } - - // Load the Static Pod spec in order to be able to create a self-hosted variant of that file - podSpec, err := loadPodSpecFromFile(manifestPath) - if err != nil { - return err - } - - // Build a DaemonSet object from the loaded PodSpec - ds := BuildDaemonSet(componentName, podSpec, mutators) - - // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out - if err := apiclient.TryRunCommand(func() error { - return apiclient.CreateOrUpdateDaemonSet(client, ds) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil { - return err - } - - // Remove the old Static Pod manifest if not dryrunning - if !dryRun { - if err := os.RemoveAll(manifestPath); err != nil { - return errors.Wrapf(err, "unable to delete static pod manifest for %s ", componentName) - } - } - - // Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to - // remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy, - // because this blocks until the API server returns a 404 Not Found when getting the Static Pod - staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name) - if err := waiter.WaitForPodToDisappear(staticPodName); err != nil { - return err - } - - // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above) - if err := waiter.WaitForAPI(); err != nil { - return err - } - - fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds()) - } - return nil -} - -// BuildDaemonSet is responsible for mutating the PodSpec and returns a DaemonSet which is suitable for self-hosting -func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *apps.DaemonSet { - - // Mutate the PodSpec so it's suitable for self-hosting - mutatePodSpec(mutators, name, podSpec) - - // Return a DaemonSet based on that Spec - return &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.AddSelfHostedPrefix(name), - Namespace: metav1.NamespaceSystem, - Labels: BuildSelfhostedComponentLabels(name), - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: BuildSelfhostedComponentLabels(name), - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: BuildSelfhostedComponentLabels(name), - }, - Spec: *podSpec, - }, - UpdateStrategy: apps.DaemonSetUpdateStrategy{ - // Make the DaemonSet utilize the RollingUpdate rollout strategy - Type: apps.RollingUpdateDaemonSetStrategyType, - }, - }, - } -} - -// BuildSelfhostedComponentLabels returns the labels for a self-hosted component -func BuildSelfhostedComponentLabels(component string) map[string]string { - return map[string]string{ - "k8s-app": kubeadmconstants.AddSelfHostedPrefix(component), - } -} - -// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod -func BuildSelfHostedComponentLabelQuery(componentName string) string { - return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName)) -} - -func loadPodSpecFromFile(filePath string) (*v1.PodSpec, error) { - podDef, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to read file path %s", filePath) - } - - if len(podDef) == 0 { - return nil, errors.Errorf("file was empty: %s", filePath) - } - - codec := clientscheme.Codecs.UniversalDecoder() - pod := &v1.Pod{} - - if err = runtime.DecodeInto(codec, podDef, pod); err != nil { - return nil, errors.Wrap(err, "failed decoding pod") - } - - return &pod.Spec, nil -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go deleted file mode 100644 index e64db5007ce..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ /dev/null @@ -1,610 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "bytes" - "io/ioutil" - "os" - "testing" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - testAPIServerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-apiserver - namespace: kube-system -spec: - containers: - - command: - - kube-apiserver - - --service-account-key-file=/etc/kubernetes/pki/sa.pub - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --secure-port=6443 - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-group-headers=X-Remote-Group - - --service-cluster-ip-range=10.96.0.0/12 - - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --advertise-address=192.168.1.115 - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - - --insecure-port=0 - - --experimental-bootstrap-token-auth=true - - --requestheader-username-headers=X-Remote-User - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-allowed-names=front-proxy-client - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --allow-privileged=true - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --authorization-mode=Node,RBAC - - --etcd-servers=http://127.0.0.1:2379 - image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 6443 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-apiserver - resources: - requests: - cpu: 250m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki -status: {} -` - - testAPIServerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-apiserver - name: self-hosted-kube-apiserver - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-apiserver - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-apiserver - spec: - containers: - - command: - - kube-apiserver - - --service-account-key-file=/etc/kubernetes/pki/sa.pub - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --secure-port=6443 - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-group-headers=X-Remote-Group - - --service-cluster-ip-range=10.96.0.0/12 - - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --advertise-address=$(HOST_IP) - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - - --insecure-port=0 - - --experimental-bootstrap-token-auth=true - - --requestheader-username-headers=X-Remote-User - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-allowed-names=front-proxy-client - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --allow-privileged=true - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --authorization-mode=Node,RBAC - - --etcd-servers=http://127.0.0.1:2379 - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 6443 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-apiserver - resources: - requests: - cpu: 250m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` - - testControllerManagerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-controller-manager - namespace: kube-system -spec: - containers: - - command: - - kube-controller-manager - - --leader-elect=true - - --controllers=*,bootstrapsigner,tokencleaner - - --kubeconfig=/etc/kubernetes/controller-manager.conf - - --root-ca-file=/etc/kubernetes/pki/ca.crt - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --bind-address=127.0.0.1 - - --use-service-account-credentials=true - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10257 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/kubernetes/controller-manager.conf - name: kubeconfig - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/kubernetes/controller-manager.conf - type: FileOrCreate - name: kubeconfig - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki -status: {} -` - - testControllerManagerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-controller-manager - name: self-hosted-kube-controller-manager - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-controller-manager - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-controller-manager - spec: - containers: - - command: - - kube-controller-manager - - --leader-elect=true - - --controllers=*,bootstrapsigner,tokencleaner - - --kubeconfig=/etc/kubernetes/controller-manager.conf - - --root-ca-file=/etc/kubernetes/pki/ca.crt - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --bind-address=127.0.0.1 - - --use-service-account-credentials=true - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10257 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/kubernetes/controller-manager.conf - name: kubeconfig - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/kubernetes/controller-manager.conf - type: FileOrCreate - name: kubeconfig - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` - - testSchedulerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-scheduler - namespace: kube-system -spec: - containers: - - command: - - kube-scheduler - - --leader-elect=true - - --kubeconfig=/etc/kubernetes/scheduler.conf - - --bind-address=127.0.0.1 - image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-scheduler - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /etc/kubernetes/scheduler.conf - name: kubeconfig - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/scheduler.conf - type: FileOrCreate - name: kubeconfig -status: {} -` - - testSchedulerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-scheduler - name: self-hosted-kube-scheduler - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-scheduler - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-scheduler - spec: - containers: - - command: - - kube-scheduler - - --leader-elect=true - - --kubeconfig=/etc/kubernetes/scheduler.conf - - --bind-address=127.0.0.1 - image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-scheduler - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /etc/kubernetes/scheduler.conf - name: kubeconfig - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/scheduler.conf - type: FileOrCreate - name: kubeconfig - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` -) - -func TestBuildDaemonSet(t *testing.T) { - var tests = []struct { - component string - podBytes []byte - dsBytes []byte - }{ - { - component: constants.KubeAPIServer, - podBytes: []byte(testAPIServerPod), - dsBytes: []byte(testAPIServerDaemonSet), - }, - { - component: constants.KubeControllerManager, - podBytes: []byte(testControllerManagerPod), - dsBytes: []byte(testControllerManagerDaemonSet), - }, - { - component: constants.KubeScheduler, - podBytes: []byte(testSchedulerPod), - dsBytes: []byte(testSchedulerDaemonSet), - }, - } - - for _, rt := range tests { - t.Run(rt.component, func(t *testing.T) { - tempFile, err := createTempFileWithContent(rt.podBytes) - if err != nil { - t.Errorf("error creating tempfile with content:%v", err) - } - defer os.Remove(tempFile) - - podSpec, err := loadPodSpecFromFile(tempFile) - if err != nil { - t.Fatalf("couldn't load the specified Pod Spec") - } - - ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators()) - dsBytes, err := kubeadmutil.MarshalToYaml(ds, apps.SchemeGroupVersion) - if err != nil { - t.Fatalf("failed to marshal daemonset to YAML: %v", err) - } - - if !bytes.Equal(dsBytes, rt.dsBytes) { - t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes) - } - }) - } -} - -func TestLoadPodSpecFromFile(t *testing.T) { - tests := []struct { - name string - content string - expectError bool - }{ - { - name: "no content", - content: "", - expectError: true, - }, - { - name: "valid YAML", - content: ` -apiVersion: v1 -kind: Pod -metadata: - name: testpod -spec: - containers: - - image: k8s.gcr.io/busybox -`, - expectError: false, - }, - { - name: "valid JSON", - content: ` -{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": "testpod" - }, - "spec": { - "containers": [ - { - "image": "k8s.gcr.io/busybox" - } - ] - } -}`, - expectError: false, - }, - { - name: "incorrect PodSpec", - content: ` -apiVersion: v1 -kind: Pod -metadata: - name: testpod -spec: - - image: k8s.gcr.io/busybox -`, - expectError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - tempFile, err := createTempFileWithContent([]byte(rt.content)) - if err != nil { - t.Errorf("error creating tempfile with content:%v", err) - } - defer os.Remove(tempFile) - - _, err = loadPodSpecFromFile(tempFile) - if (err != nil) != rt.expectError { - t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err) - } - }) - } - - t.Run("empty file name", func(t *testing.T) { - _, err := loadPodSpecFromFile("") - if err == nil { - t.Error("unexpected success: loadPodSpecFromFile should return error when no file is given") - } - }) -} - -func createTempFileWithContent(content []byte) (string, error) { - tempFile, err := ioutil.TempFile("", "") - if err != nil { - return "", errors.Wrap(err, "cannot create temporary file") - } - if _, err = tempFile.Write([]byte(content)); err != nil { - return "", errors.Wrap(err, "cannot save temporary file") - } - if err = tempFile.Close(); err != nil { - return "", errors.Wrap(err, "cannot close temporary file") - } - return tempFile.Name(), nil -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go deleted file mode 100644 index a877ec34e53..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go +++ /dev/null @@ -1,356 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strings" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -type tlsKeyPairPath struct { - name string - cert string - key string -} - -func apiServerCertificatesVolumeSource() v1.VolumeSource { - return v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.CACertName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerKubeletClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerKubeletClientKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.ServiceAccountPublicKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyCACertName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.FrontProxyClientKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1), - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.EtcdCACertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.EtcdCAKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerEtcdClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerEtcdClientKeyName, - }, - }, - }, - }, - }, - }, - } -} - -func controllerManagerCertificatesVolumeSource() v1.VolumeSource { - return v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.CACertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.CAKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.ServiceAccountPrivateKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyCACertName, - }, - }, - }, - }, - }, - }, - } -} - -func kubeConfigVolumeSource(kubeconfigSecretName string) v1.VolumeSource { - return v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: strings.Replace(kubeconfigSecretName, "/", "-", -1), - }, - } -} - -func uploadTLSSecrets(client clientset.Interface, certDir string) error { - for _, tlsKeyPair := range getTLSKeyPairs() { - secret, err := createTLSSecretFromFiles( - tlsKeyPair.name, - filepath.Join(certDir, tlsKeyPair.cert), - filepath.Join(certDir, tlsKeyPair.key), - ) - if err != nil { - return err - } - - if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { - return err - } - fmt.Printf("[self-hosted] Created TLS secret %q from %s and %s\n", tlsKeyPair.name, tlsKeyPair.cert, tlsKeyPair.key) - } - - return nil -} - -func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error { - files := []string{ - kubeadmconstants.SchedulerKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - } - for _, file := range files { - kubeConfigPath := filepath.Join(kubeConfigDir, file) - secret, err := createOpaqueSecretFromFile(file, kubeConfigPath) - if err != nil { - return err - } - - if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { - return err - } - fmt.Printf("[self-hosted] Created secret for kubeconfig file %q\n", file) - } - - return nil -} - -func createTLSSecretFromFiles(secretName, crt, key string) (*v1.Secret, error) { - crtBytes, err := ioutil.ReadFile(crt) - if err != nil { - return nil, err - } - keyBytes, err := ioutil.ReadFile(key) - if err != nil { - return nil, err - } - - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: metav1.NamespaceSystem, - }, - Type: v1.SecretTypeTLS, - Data: map[string][]byte{ - v1.TLSCertKey: crtBytes, - v1.TLSPrivateKeyKey: keyBytes, - }, - }, nil -} - -func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) { - fileBytes, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: metav1.NamespaceSystem, - }, - Type: v1.SecretTypeOpaque, - Data: map[string][]byte{ - filepath.Base(file): fileBytes, - }, - }, nil -} - -func getTLSKeyPairs() []*tlsKeyPairPath { - return []*tlsKeyPairPath{ - { - name: kubeadmconstants.CACertAndKeyBaseName, - cert: kubeadmconstants.CACertName, - key: kubeadmconstants.CAKeyName, - }, - { - name: kubeadmconstants.APIServerCertAndKeyBaseName, - cert: kubeadmconstants.APIServerCertName, - key: kubeadmconstants.APIServerKeyName, - }, - { - name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - cert: kubeadmconstants.APIServerKubeletClientCertName, - key: kubeadmconstants.APIServerKubeletClientKeyName, - }, - { - name: kubeadmconstants.ServiceAccountKeyBaseName, - cert: kubeadmconstants.ServiceAccountPublicKeyName, - key: kubeadmconstants.ServiceAccountPrivateKeyName, - }, - { - name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - cert: kubeadmconstants.FrontProxyCACertName, - key: kubeadmconstants.FrontProxyCAKeyName, - }, - { - name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - cert: kubeadmconstants.FrontProxyClientCertName, - key: kubeadmconstants.FrontProxyClientKeyName, - }, - { - name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1), - cert: kubeadmconstants.EtcdCACertName, - key: kubeadmconstants.EtcdCAKeyName, - }, - { - name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, - cert: kubeadmconstants.APIServerEtcdClientCertName, - key: kubeadmconstants.APIServerEtcdClientKeyName, - }, - } -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go deleted file mode 100644 index bf2e51c1f16..00000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "io/ioutil" - "log" - "os" - "testing" -) - -func createTemporaryFile(name string) *os.File { - content := []byte("foo") - tmpfile, err := ioutil.TempFile("", name) - if err != nil { - log.Fatal(err) - } - - if _, err := tmpfile.Write(content); err != nil { - log.Fatal(err) - } - - return tmpfile -} - -func TestCreateTLSSecretFromFile(t *testing.T) { - tmpCert := createTemporaryFile("foo.crt") - defer os.Remove(tmpCert.Name()) - tmpKey := createTemporaryFile("foo.key") - defer os.Remove(tmpKey.Name()) - - _, err := createTLSSecretFromFiles("foo", tmpCert.Name(), tmpKey.Name()) - if err != nil { - log.Fatal(err) - } - - if err := tmpCert.Close(); err != nil { - log.Fatal(err) - } - - if err := tmpKey.Close(); err != nil { - log.Fatal(err) - } -} - -func TestCreateOpaqueSecretFromFile(t *testing.T) { - tmpFile := createTemporaryFile("foo") - defer os.Remove(tmpFile.Name()) - - _, err := createOpaqueSecretFromFile("foo", tmpFile.Name()) - if err != nil { - log.Fatal(err) - } - - if err := tmpFile.Close(); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index dbe8d91c0a7..758e4aa140a 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -35,7 +35,6 @@ go_library( "//cmd/kubeadm/app/util/etcd:go_default_library", "//cmd/kubeadm/app/util/image:go_default_library", "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go index 3a238968d90..1f54cbe2d2d 100644 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,11 +32,12 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - utilpointer "k8s.io/utils/pointer" ) // healthCheck is a helper struct for easily performing healthchecks against the cluster and printing the output @@ -257,49 +257,6 @@ func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfigu return errors.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests) } -// IsControlPlaneSelfHosted returns whether the control plane is self hosted or not -func IsControlPlaneSelfHosted(client clientset.Interface) bool { - notReadyDaemonSets, err := getNotReadyDaemonSets(client) - if err != nil { - return false - } - - // If there are no NotReady DaemonSets, we are using selfhosting - return len(notReadyDaemonSets) == 0 -} - -// getNotReadyDaemonSets gets the amount of Ready control plane DaemonSets -func getNotReadyDaemonSets(client clientset.Interface) ([]error, error) { - notReadyDaemonSets := []error{} - for _, component := range constants.ControlPlaneComponents { - dsName := constants.AddSelfHostedPrefix(component) - ds, err := client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.TODO(), dsName, metav1.GetOptions{}) - if err != nil { - return nil, errors.Errorf("couldn't get daemonset %q in the %s namespace", dsName, metav1.NamespaceSystem) - } - - if err := daemonSetHealth(&ds.Status); err != nil { - notReadyDaemonSets = append(notReadyDaemonSets, errors.Wrapf(err, "DaemonSet %q not healthy", dsName)) - } - } - return notReadyDaemonSets, nil -} - -// daemonSetHealth is a helper function for getting the health of a DaemonSet's status -func daemonSetHealth(dsStatus *apps.DaemonSetStatus) error { - if dsStatus.CurrentNumberScheduled != dsStatus.DesiredNumberScheduled { - return errors.Errorf("current number of scheduled Pods ('%d') doesn't match the amount of desired Pods ('%d')", - dsStatus.CurrentNumberScheduled, dsStatus.DesiredNumberScheduled) - } - if dsStatus.NumberAvailable == 0 { - return errors.New("no available Pods for DaemonSet") - } - if dsStatus.NumberReady == 0 { - return errors.New("no ready Pods for DaemonSet") - } - return nil -} - // getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady func getNotReadyNodes(nodes []v1.Node) []string { notReadyNodes := []string{}