github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 02:11:09 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #97627 from knight42/refactor/remove-kubeadm-pivot

Browse Source 
refactor(kubeadm): remove deprecated command "alpha selfhosting pivot"
This commit is contained in:
Kubernetes Prow Robot 2020-12-30 23:13:50 -08:00 committed by GitHub
commit 8725c3bf12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 2 additions and 2357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubeadm/app/BUILD
View File

@ -48,7 +48,6 @@ filegroup(
"//cmd/kubeadm/app/phases/kubelet:all-srcs",
"//cmd/kubeadm/app/phases/markcontrolplane:all-srcs",
"//cmd/kubeadm/app/phases/patchnode:all-srcs",
"//cmd/kubeadm/app/phases/selfhosting:all-srcs",
"//cmd/kubeadm/app/phases/upgrade:all-srcs",
"//cmd/kubeadm/app/phases/uploadconfig:all-srcs",
"//cmd/kubeadm/app/preflight:all-srcs",

6
cmd/kubeadm/app/cmd/alpha/BUILD
View File

@ -6,7 +6,6 @@ go_library(
"alpha.go",
"certs.go",
"kubeconfig.go",
"selfhosting.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha",
visibility = ["//visibility:public"],
@ -14,18 +13,13 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
"//cmd/kubeadm/app/cmd/options:go_default_library",
"//cmd/kubeadm/app/cmd/phases:go_default_library",
"//cmd/kubeadm/app/cmd/util:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/renewal:go_default_library",
"//cmd/kubeadm/app/phases/copycerts:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library",

6
cmd/kubeadm/app/cmd/alpha/alpha.go
View File

@ -33,12 +33,6 @@ func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command {
deprecateCommand(`please use the same command under "kubeadm kubeconfig"`, kubeconfigCmd)
cmd.AddCommand(kubeconfigCmd)
const shDeprecatedMessage = "self-hosting support in kubeadm is deprecated " +
"and will be removed in a future release"
shCommand := newCmdSelfhosting(in)
deprecateCommand(shDeprecatedMessage, shCommand)
cmd.AddCommand(shCommand)
certsCommand := NewCmdCertsUtility(out)
deprecateCommand(`please use the same command under "kubeadm certs"`, certsCommand)
cmd.AddCommand(certsCommand)

161
cmd/kubeadm/app/cmd/alpha/selfhosting.go
View File

@ -1,161 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package alpha
import (
"bufio"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)
var (
selfhostingLongDesc = cmdutil.LongDesc(`
Convert static Pod files for control plane components into self-hosted DaemonSets configured via the Kubernetes API.
See the documentation for self-hosting limitations.
` + cmdutil.AlphaDisclaimer)
selfhostingExample = cmdutil.Examples(`
# Convert a static Pod-hosted control plane into a self-hosted one.
kubeadm alpha phase self-hosting convert-from-staticpods
`)
)
// newCmdSelfhosting returns the self-hosting Cobra command
func newCmdSelfhosting(in io.Reader) *cobra.Command {
cmd := &cobra.Command{
Use: "selfhosting",
Aliases: []string{"selfhosted", "self-hosting"},
Short: "Make a kubeadm cluster self-hosted",
Long: cmdutil.MacroCommandLongDescription,
}
cmd.AddCommand(getSelfhostingSubCommand(in))
return cmd
}
// getSelfhostingSubCommand returns sub commands for Self-hosting phase
func getSelfhostingSubCommand(in io.Reader) *cobra.Command {
cfg := &kubeadmapiv1beta2.ClusterConfiguration{}
// Default values for the cobra help text
kubeadmscheme.Scheme.Default(cfg)
var cfgPath, featureGatesString, kubeConfigFile string
forcePivot, certsInSecrets := false, false
// Creates the UX Command
cmd := &cobra.Command{
Use: "pivot",
Aliases: []string{"from-staticpods"},
Short: "Convert a static Pod-hosted control plane into a self-hosted one",
Long: selfhostingLongDesc,
Example: selfhostingExample,
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if !forcePivot {
fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!")
fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ")
s := bufio.NewScanner(in)
s.Scan()
if err = s.Err(); err != nil {
return err
}
if strings.ToLower(s.Text()) != "y" {
return errors.New("aborted pivot operation")
}
}
fmt.Println("[pivot] pivoting cluster to self-hosted")
if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil {
return err
}
if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
return err
}
// Gets the Kubernetes client
kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile)
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
if err != nil {
return err
}
// KubernetesVersion is not used, but we set it explicitly to avoid the lookup
// of the version from the internet when executing LoadOrDefaultInitConfiguration
phases.SetKubernetesVersion(cfg)
// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, cfg)
if err != nil {
return err
}
// Converts the Static Pod-hosted control plane into a self-hosted one
waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout)
return selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets)
},
Args: cobra.NoArgs,
}
// Add flags to the command
// flags bound to the configuration object
cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`)
options.AddConfigFlag(cmd.Flags(), &cfgPath)
cmd.Flags().BoolVarP(
&certsInSecrets, "store-certs-in-secrets", "s",
false, "Enable storing certs in secrets")
cmd.Flags().BoolVarP(
&forcePivot, "force", "f", false,
"Pivot the cluster without prompting for confirmation",
)
// flags that are not bound to the configuration object
// Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile)
return cmd
}

5
cmd/kubeadm/app/cmd/upgrade/common.go
View File

@ -127,11 +127,6 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr
return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath)
}
// Check if the cluster is self-hosted
if upgrade.IsControlPlaneSelfHosted(client) {
return nil, nil, nil, errors.New("cannot upgrade a self-hosted control plane")
}
// Fetch the configuration from a file or ConfigMap and validate it
fmt.Println("[upgrade/config] Making sure the configuration is correct:")

8
cmd/kubeadm/app/constants/constants.go
View File

@ -304,9 +304,6 @@ const (
// Kubelet defines variable used internally when referring to the Kubelet
Kubelet = "kubelet"
// SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has
SelfHostingPrefix = "self-hosted-"
// KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume)
KubeCertificatesVolumeName = "k8s-certs"
@ -555,11 +552,6 @@ func GetKubeletKubeConfigPath() string {
return filepath.Join(KubernetesDir, KubeletKubeConfigFileName)
}
// AddSelfHostedPrefix adds the self-hosted- prefix to the component name
func AddSelfHostedPrefix(componentName string) string {
return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName)
}
// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous)
func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) {
tempDir := path.Join(KubernetesDir, TempDirForKubeadm)

35
cmd/kubeadm/app/constants/constants_test.go
View File

@ -110,41 +110,6 @@ func TestGetStaticPodFilepath(t *testing.T) {
}
}
func TestAddSelfHostedPrefix(t *testing.T) {
var tests = []struct {
componentName, expected string
}{
{
componentName: "kube-apiserver",
expected: "self-hosted-kube-apiserver",
},
{
componentName: "kube-controller-manager",
expected: "self-hosted-kube-controller-manager",
},
{
componentName: "kube-scheduler",
expected: "self-hosted-kube-scheduler",
},
{
componentName: "foo",
expected: "self-hosted-foo",
},
}
for _, rt := range tests {
t.Run(rt.componentName, func(t *testing.T) {
actual := AddSelfHostedPrefix(rt.componentName)
if actual != rt.expected {
t.Errorf(
"failed AddSelfHostedPrefix:\n\texpected: %s\n\t actual: %s",
rt.expected,
actual,
)
}
})
}
}
func TestEtcdSupportedVersion(t *testing.T) {
var supportedEtcdVersion = map[uint8]string{
13: "3.2.24",

61
cmd/kubeadm/app/phases/selfhosting/BUILD
View File

@ -1,61 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"podspec_mutation_test.go",
"selfhosting_test.go",
"selfhosting_volumes_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"podspec_mutation.go",
"selfhosting.go",
"selfhosting_volumes.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

205
cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go
View File

@ -1,205 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"path/filepath"
"strings"
v1 "k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
const (
// selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted
// Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from
// the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki)
selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig"
)
// PodSpecMutatorFunc is a function capable of mutating a PodSpec
type PodSpecMutatorFunc func(*v1.PodSpec)
// GetDefaultMutators gets the mutator functions that always should be used
func GetDefaultMutators() map[string][]PodSpecMutatorFunc {
return map[string][]PodSpecMutatorFunc{
kubeadmconstants.KubeAPIServer: {
addNodeSelectorToPodSpec,
setControlPlaneTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
setHostIPOnPodSpec,
},
kubeadmconstants.KubeControllerManager: {
addNodeSelectorToPodSpec,
setControlPlaneTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
kubeadmconstants.KubeScheduler: {
addNodeSelectorToPodSpec,
setControlPlaneTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
}
}
// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed
func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := GetDefaultMutators()
if certsInSecrets {
// Some extra work to be done if we should store the control plane certificates in Secrets
// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer)
mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager)
mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler)
}
return mutators
}
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) {
// Get the mutator functions for the component in question, then loop through and execute them
mutatorsForComponent := mutators[name]
for _, mutateFunc := range mutatorsForComponent {
mutateFunc(podSpec)
}
}
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the control-plane label
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
if podSpec.NodeSelector == nil {
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleOldControlPlane: ""}
return
}
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleOldControlPlane] = ""
}
// setControlPlaneTolerationOnPodSpec makes the Pod tolerate the control-plane taint
func setControlPlaneTolerationOnPodSpec(podSpec *v1.PodSpec) {
if podSpec.Tolerations == nil {
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.OldControlPlaneToleration}
return
}
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.OldControlPlaneToleration)
}
// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API
func setHostIPOnPodSpec(podSpec *v1.PodSpec) {
envVar := v1.EnvVar{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
}
podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar)
for i := range podSpec.Containers[0].Command {
if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") {
podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)"
}
}
}
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource()
}
}
}
// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource()
} else if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
controllerManagerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
argMap["kubeconfig"] = controllerManagerKubeConfigPath
if _, ok := argMap["authentication-kubeconfig"]; ok {
argMap["authentication-kubeconfig"] = controllerManagerKubeConfigPath
}
if _, ok := argMap["authorization-kubeconfig"]; ok {
argMap["authorization-kubeconfig"] = controllerManagerKubeConfigPath
}
return argMap
})
}
// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
schedulerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName)
argMap["kubeconfig"] = schedulerKubeConfigPath
if _, ok := argMap["authentication-kubeconfig"]; ok {
argMap["authentication-kubeconfig"] = schedulerKubeConfigPath
}
if _, ok := argMap["authorization-kubeconfig"]; ok {
argMap["authorization-kubeconfig"] = schedulerKubeConfigPath
}
return argMap
})
}

590
cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go
View File

@ -1,590 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"reflect"
"sort"
"testing"
v1 "k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func TestMutatePodSpec(t *testing.T) {
var tests = []struct {
name string
component string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "mutate api server podspec",
component: kubeadmconstants.KubeAPIServer,
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=10.0.0.1",
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=$(HOST_IP)",
},
Env: []v1.EnvVar{
{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
},
},
},
},
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.OldControlPlaneToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
name: "mutate controller manager podspec",
component: kubeadmconstants.KubeControllerManager,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.OldControlPlaneToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
name: "mutate scheduler podspec",
component: kubeadmconstants.KubeScheduler,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.OldControlPlaneToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestAddNodeSelectorToPodSpec(t *testing.T) {
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "empty podspec",
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
},
},
},
{
name: "podspec with a valid node selector",
podSpec: &v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
},
expected: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
addNodeSelectorToPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetControlPlaneTolerationOnPodSpec(t *testing.T) {
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "empty podspec",
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
kubeadmconstants.OldControlPlaneToleration,
},
},
},
{
name: "podspec with a valid toleration",
podSpec: &v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
},
},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
kubeadmconstants.OldControlPlaneToleration,
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setControlPlaneTolerationOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setControlPlaneTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetRightDNSPolicyOnPodSpec(t *testing.T) {
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "empty podspec",
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
name: "podspec with a v1.DNSClusterFirst policy",
podSpec: &v1.PodSpec{
DNSPolicy: v1.DNSClusterFirst,
},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setRightDNSPolicyOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetHostIPOnPodSpec(t *testing.T) {
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "set HOST_IP env var on a podspec",
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=10.0.0.1",
},
Env: []v1.EnvVar{},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=$(HOST_IP)",
},
Env: []v1.EnvVar{
{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
},
},
},
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setHostIPOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setHostIPOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetSelfHostedVolumesForAPIServer(t *testing.T) {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "set selfhosted volumes for api server",
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
},
Command: []string{
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
},
Command: []string{
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: apiServerCertificatesVolumeSource(),
},
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setSelfHostedVolumesForAPIServer(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForAPIServer:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetSelfHostedVolumesForControllerManager(t *testing.T) {
hostPathFileOrCreate := v1.HostPathFileOrCreate
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "set selfhosted volumes for controller mananger",
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/controller-manager.conf",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/controller-manager.conf",
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/controller-manager.conf",
Type: &hostPathFileOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/kubeconfig",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf",
"--authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf",
"--authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: controllerManagerCertificatesVolumeSource(),
},
{
Name: "kubeconfig",
VolumeSource: kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName),
},
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setSelfHostedVolumesForControllerManager(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForControllerManager:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}
func TestSetSelfHostedVolumesForScheduler(t *testing.T) {
hostPathFileOrCreate := v1.HostPathFileOrCreate
var tests = []struct {
name string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
name: "set selfhosted volumes for scheduler",
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/scheduler.conf",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/scheduler.conf",
"--authentication-kubeconfig=/etc/kubernetes/scheduler.conf",
"--authorization-kubeconfig=/etc/kubernetes/scheduler.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/scheduler.conf",
Type: &hostPathFileOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/kubeconfig",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf",
"--authentication-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf",
"--authorization-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "kubeconfig",
VolumeSource: kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName),
},
},
},
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
setSelfHostedVolumesForScheduler(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForScheduler:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
})
}
}

195
cmd/kubeadm/app/phases/selfhosting/selfhosting.go
View File

@ -1,195 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"fmt"
"io/ioutil"
"os"
"time"
"k8s.io/klog/v2"
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
const (
// selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out
selfHostingWaitTimeout = 2 * time.Minute
// selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets
selfHostingFailureThreshold int = 5
)
// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
// It achieves that task this way:
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
// 2. Extract the PodSpec from that Static Pod specification
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
// 5. Create the DaemonSet resource. Wait until the Pods are running.
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
// 7. The self-hosted containers should now step up and take over.
// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue
// Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error {
klog.V(1).Infoln("creating self hosted control plane")
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)
// Here the map of different mutators to use for the control plane's PodSpec is stored
klog.V(1).Infoln("getting mutators")
mutators := GetMutatorsFromFeatureGates(certsInSecrets)
if certsInSecrets {
// Upload the certificates and kubeconfig files from disk to the cluster as Secrets
if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil {
return err
}
if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil {
return err
}
}
for _, componentName := range kubeadmconstants.ControlPlaneComponents {
start := time.Now()
manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir)
// Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist
if _, err := os.Stat(manifestPath); err != nil {
fmt.Printf("[self-hosted] The Static Pod for the component %q doesn't seem to be on the disk; trying the next one\n", componentName)
continue
}
// Load the Static Pod spec in order to be able to create a self-hosted variant of that file
podSpec, err := loadPodSpecFromFile(manifestPath)
if err != nil {
return err
}
// Build a DaemonSet object from the loaded PodSpec
ds := BuildDaemonSet(componentName, podSpec, mutators)
// Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out
if err := apiclient.TryRunCommand(func() error {
return apiclient.CreateOrUpdateDaemonSet(client, ds)
}, selfHostingFailureThreshold); err != nil {
return err
}
// Wait for the self-hosted component to come up
if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil {
return err
}
// Remove the old Static Pod manifest if not dryrunning
if !dryRun {
if err := os.RemoveAll(manifestPath); err != nil {
return errors.Wrapf(err, "unable to delete static pod manifest for %s ", componentName)
}
}
// Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy,
// because this blocks until the API server returns a 404 Not Found when getting the Static Pod
staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name)
if err := waiter.WaitForPodToDisappear(staticPodName); err != nil {
return err
}
// Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above)
if err := waiter.WaitForAPI(); err != nil {
return err
}
fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
}
return nil
}
// BuildDaemonSet is responsible for mutating the PodSpec and returns a DaemonSet which is suitable for self-hosting
func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *apps.DaemonSet {
// Mutate the PodSpec so it's suitable for self-hosting
mutatePodSpec(mutators, name, podSpec)
// Return a DaemonSet based on that Spec
return &apps.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.AddSelfHostedPrefix(name),
Namespace: metav1.NamespaceSystem,
Labels: BuildSelfhostedComponentLabels(name),
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: BuildSelfhostedComponentLabels(name),
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: BuildSelfhostedComponentLabels(name),
},
Spec: *podSpec,
},
UpdateStrategy: apps.DaemonSetUpdateStrategy{
// Make the DaemonSet utilize the RollingUpdate rollout strategy
Type: apps.RollingUpdateDaemonSetStrategyType,
},
},
}
}
// BuildSelfhostedComponentLabels returns the labels for a self-hosted component
func BuildSelfhostedComponentLabels(component string) map[string]string {
return map[string]string{
"k8s-app": kubeadmconstants.AddSelfHostedPrefix(component),
}
}
// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod
func BuildSelfHostedComponentLabelQuery(componentName string) string {
return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName))
}
func loadPodSpecFromFile(filePath string) (*v1.PodSpec, error) {
podDef, err := ioutil.ReadFile(filePath)
if err != nil {
return nil, errors.Wrapf(err, "failed to read file path %s", filePath)
}
if len(podDef) == 0 {
return nil, errors.Errorf("file was empty: %s", filePath)
}
codec := clientscheme.Codecs.UniversalDecoder()
pod := &v1.Pod{}
if err = runtime.DecodeInto(codec, podDef, pod); err != nil {
return nil, errors.Wrap(err, "failed decoding pod")
}
return &pod.Spec, nil
}

610
cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go
View File

@ -1,610 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
const (
testAPIServerPod = `
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-apiserver
namespace: kube-system
spec:
containers:
- command:
- kube-apiserver
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --secure-port=6443
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-group-headers=X-Remote-Group
- --service-cluster-ip-range=10.96.0.0/12
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --advertise-address=192.168.1.115
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --insecure-port=0
- --experimental-bootstrap-token-auth=true
- --requestheader-username-headers=X-Remote-User
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-allowed-names=front-proxy-client
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --allow-privileged=true
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --authorization-mode=Node,RBAC
- --etcd-servers=http://127.0.0.1:2379
image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
status: {}
`
testAPIServerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
name: self-hosted-kube-apiserver
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-apiserver
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
spec:
containers:
- command:
- kube-apiserver
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --secure-port=6443
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-group-headers=X-Remote-Group
- --service-cluster-ip-range=10.96.0.0/12
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --advertise-address=$(HOST_IP)
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --insecure-port=0
- --experimental-bootstrap-token-auth=true
- --requestheader-username-headers=X-Remote-User
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-allowed-names=front-proxy-client
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --allow-privileged=true
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --authorization-mode=Node,RBAC
- --etcd-servers=http://127.0.0.1:2379
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
priorityClassName: system-cluster-critical
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testControllerManagerPod = `
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --leader-elect=true
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --bind-address=127.0.0.1
- --use-service-account-credentials=true
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10257
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/kubernetes/controller-manager.conf
type: FileOrCreate
name: kubeconfig
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
status: {}
`
testControllerManagerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
name: self-hosted-kube-controller-manager
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-controller-manager
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
spec:
containers:
- command:
- kube-controller-manager
- --leader-elect=true
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --bind-address=127.0.0.1
- --use-service-account-credentials=true
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10257
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
priorityClassName: system-cluster-critical
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/kubernetes/controller-manager.conf
type: FileOrCreate
name: kubeconfig
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testSchedulerPod = `
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-scheduler
namespace: kube-system
spec:
containers:
- command:
- kube-scheduler
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --bind-address=127.0.0.1
image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10259
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes/scheduler.conf
name: kubeconfig
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/kubernetes/scheduler.conf
type: FileOrCreate
name: kubeconfig
status: {}
`
testSchedulerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
name: self-hosted-kube-scheduler
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-scheduler
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
spec:
containers:
- command:
- kube-scheduler
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --bind-address=127.0.0.1
image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10259
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes/scheduler.conf
name: kubeconfig
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
priorityClassName: system-cluster-critical
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/scheduler.conf
type: FileOrCreate
name: kubeconfig
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
)
func TestBuildDaemonSet(t *testing.T) {
var tests = []struct {
component string
podBytes []byte
dsBytes []byte
}{
{
component: constants.KubeAPIServer,
podBytes: []byte(testAPIServerPod),
dsBytes: []byte(testAPIServerDaemonSet),
},
{
component: constants.KubeControllerManager,
podBytes: []byte(testControllerManagerPod),
dsBytes: []byte(testControllerManagerDaemonSet),
},
{
component: constants.KubeScheduler,
podBytes: []byte(testSchedulerPod),
dsBytes: []byte(testSchedulerDaemonSet),
},
}
for _, rt := range tests {
t.Run(rt.component, func(t *testing.T) {
tempFile, err := createTempFileWithContent(rt.podBytes)
if err != nil {
t.Errorf("error creating tempfile with content:%v", err)
}
defer os.Remove(tempFile)
podSpec, err := loadPodSpecFromFile(tempFile)
if err != nil {
t.Fatalf("couldn't load the specified Pod Spec")
}
ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators())
dsBytes, err := kubeadmutil.MarshalToYaml(ds, apps.SchemeGroupVersion)
if err != nil {
t.Fatalf("failed to marshal daemonset to YAML: %v", err)
}
if !bytes.Equal(dsBytes, rt.dsBytes) {
t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
}
})
}
}
func TestLoadPodSpecFromFile(t *testing.T) {
tests := []struct {
name string
content string
expectError bool
}{
{
name: "no content",
content: "",
expectError: true,
},
{
name: "valid YAML",
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
containers:
- image: k8s.gcr.io/busybox
`,
expectError: false,
},
{
name: "valid JSON",
content: `
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "testpod"
},
"spec": {
"containers": [
{
"image": "k8s.gcr.io/busybox"
}
]
}
}`,
expectError: false,
},
{
name: "incorrect PodSpec",
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
- image: k8s.gcr.io/busybox
`,
expectError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
tempFile, err := createTempFileWithContent([]byte(rt.content))
if err != nil {
t.Errorf("error creating tempfile with content:%v", err)
}
defer os.Remove(tempFile)
_, err = loadPodSpecFromFile(tempFile)
if (err != nil) != rt.expectError {
t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err)
}
})
}
t.Run("empty file name", func(t *testing.T) {
_, err := loadPodSpecFromFile("")
if err == nil {
t.Error("unexpected success: loadPodSpecFromFile should return error when no file is given")
}
})
}
func createTempFileWithContent(content []byte) (string, error) {
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return "", errors.Wrap(err, "cannot create temporary file")
}
if _, err = tempFile.Write([]byte(content)); err != nil {
return "", errors.Wrap(err, "cannot save temporary file")
}
if err = tempFile.Close(); err != nil {
return "", errors.Wrap(err, "cannot close temporary file")
}
return tempFile.Name(), nil
}

356
cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go
View File

@ -1,356 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"fmt"
"io/ioutil"
"path/filepath"
"strings"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
type tlsKeyPairPath struct {
name string
cert string
key string
}
func apiServerCertificatesVolumeSource() v1.VolumeSource {
return v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: []v1.VolumeProjection{
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.CACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.CACertName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.APIServerCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.APIServerCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.APIServerKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.APIServerKubeletClientCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.APIServerKubeletClientKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.ServiceAccountKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.ServiceAccountPublicKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.FrontProxyCACertName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.FrontProxyClientCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.FrontProxyClientKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1),
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.EtcdCACertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.EtcdCAKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.APIServerEtcdClientCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.APIServerEtcdClientKeyName,
},
},
},
},
},
},
}
}
func controllerManagerCertificatesVolumeSource() v1.VolumeSource {
return v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: []v1.VolumeProjection{
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.CACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.CACertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.CAKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.ServiceAccountKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.ServiceAccountPrivateKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.FrontProxyCACertName,
},
},
},
},
},
},
}
}
func kubeConfigVolumeSource(kubeconfigSecretName string) v1.VolumeSource {
return v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: strings.Replace(kubeconfigSecretName, "/", "-", -1),
},
}
}
func uploadTLSSecrets(client clientset.Interface, certDir string) error {
for _, tlsKeyPair := range getTLSKeyPairs() {
secret, err := createTLSSecretFromFiles(
tlsKeyPair.name,
filepath.Join(certDir, tlsKeyPair.cert),
filepath.Join(certDir, tlsKeyPair.key),
)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil {
return err
}
fmt.Printf("[self-hosted] Created TLS secret %q from %s and %s\n", tlsKeyPair.name, tlsKeyPair.cert, tlsKeyPair.key)
}
return nil
}
func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error {
files := []string{
kubeadmconstants.SchedulerKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
}
for _, file := range files {
kubeConfigPath := filepath.Join(kubeConfigDir, file)
secret, err := createOpaqueSecretFromFile(file, kubeConfigPath)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil {
return err
}
fmt.Printf("[self-hosted] Created secret for kubeconfig file %q\n", file)
}
return nil
}
func createTLSSecretFromFiles(secretName, crt, key string) (*v1.Secret, error) {
crtBytes, err := ioutil.ReadFile(crt)
if err != nil {
return nil, err
}
keyBytes, err := ioutil.ReadFile(key)
if err != nil {
return nil, err
}
return &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: metav1.NamespaceSystem,
},
Type: v1.SecretTypeTLS,
Data: map[string][]byte{
v1.TLSCertKey: crtBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
}, nil
}
func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) {
fileBytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
return &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: metav1.NamespaceSystem,
},
Type: v1.SecretTypeOpaque,
Data: map[string][]byte{
filepath.Base(file): fileBytes,
},
}, nil
}
func getTLSKeyPairs() []*tlsKeyPairPath {
return []*tlsKeyPairPath{
{
name: kubeadmconstants.CACertAndKeyBaseName,
cert: kubeadmconstants.CACertName,
key: kubeadmconstants.CAKeyName,
},
{
name: kubeadmconstants.APIServerCertAndKeyBaseName,
cert: kubeadmconstants.APIServerCertName,
key: kubeadmconstants.APIServerKeyName,
},
{
name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
cert: kubeadmconstants.APIServerKubeletClientCertName,
key: kubeadmconstants.APIServerKubeletClientKeyName,
},
{
name: kubeadmconstants.ServiceAccountKeyBaseName,
cert: kubeadmconstants.ServiceAccountPublicKeyName,
key: kubeadmconstants.ServiceAccountPrivateKeyName,
},
{
name: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
cert: kubeadmconstants.FrontProxyCACertName,
key: kubeadmconstants.FrontProxyCAKeyName,
},
{
name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
cert: kubeadmconstants.FrontProxyClientCertName,
key: kubeadmconstants.FrontProxyClientKeyName,
},
{
name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1),
cert: kubeadmconstants.EtcdCACertName,
key: kubeadmconstants.EtcdCAKeyName,
},
{
name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
cert: kubeadmconstants.APIServerEtcdClientCertName,
key: kubeadmconstants.APIServerEtcdClientKeyName,
},
}
}

72
cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go
View File

@ -1,72 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"io/ioutil"
"log"
"os"
"testing"
)
func createTemporaryFile(name string) *os.File {
content := []byte("foo")
tmpfile, err := ioutil.TempFile("", name)
if err != nil {
log.Fatal(err)
}
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
return tmpfile
}
func TestCreateTLSSecretFromFile(t *testing.T) {
tmpCert := createTemporaryFile("foo.crt")
defer os.Remove(tmpCert.Name())
tmpKey := createTemporaryFile("foo.key")
defer os.Remove(tmpKey.Name())
_, err := createTLSSecretFromFiles("foo", tmpCert.Name(), tmpKey.Name())
if err != nil {
log.Fatal(err)
}
if err := tmpCert.Close(); err != nil {
log.Fatal(err)
}
if err := tmpKey.Close(); err != nil {
log.Fatal(err)
}
}
func TestCreateOpaqueSecretFromFile(t *testing.T) {
tmpFile := createTemporaryFile("foo")
defer os.Remove(tmpFile.Name())
_, err := createOpaqueSecretFromFile("foo", tmpFile.Name())
if err != nil {
log.Fatal(err)
}
if err := tmpFile.Close(); err != nil {
log.Fatal(err)
}
}

1
cmd/kubeadm/app/phases/upgrade/BUILD
View File

@ -35,7 +35,6 @@ go_library(
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//cmd/kubeadm/app/util/image:go_default_library",
"//cmd/kubeadm/app/util/staticpod:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/batch/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

47
cmd/kubeadm/app/phases/upgrade/health.go
View File

@ -24,7 +24,6 @@ import (
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -33,11 +32,12 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
utilpointer "k8s.io/utils/pointer"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
utilpointer "k8s.io/utils/pointer"
)
// healthCheck is a helper struct for easily performing healthchecks against the cluster and printing the output
@ -257,49 +257,6 @@ func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfigu
return errors.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests)
}
// IsControlPlaneSelfHosted returns whether the control plane is self hosted or not
func IsControlPlaneSelfHosted(client clientset.Interface) bool {
notReadyDaemonSets, err := getNotReadyDaemonSets(client)
if err != nil {
return false
}
// If there are no NotReady DaemonSets, we are using selfhosting
return len(notReadyDaemonSets) == 0
}
// getNotReadyDaemonSets gets the amount of Ready control plane DaemonSets
func getNotReadyDaemonSets(client clientset.Interface) ([]error, error) {
notReadyDaemonSets := []error{}
for _, component := range constants.ControlPlaneComponents {
dsName := constants.AddSelfHostedPrefix(component)
ds, err := client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.TODO(), dsName, metav1.GetOptions{})
if err != nil {
return nil, errors.Errorf("couldn't get daemonset %q in the %s namespace", dsName, metav1.NamespaceSystem)
}
if err := daemonSetHealth(&ds.Status); err != nil {
notReadyDaemonSets = append(notReadyDaemonSets, errors.Wrapf(err, "DaemonSet %q not healthy", dsName))
}
}
return notReadyDaemonSets, nil
}
// daemonSetHealth is a helper function for getting the health of a DaemonSet's status
func daemonSetHealth(dsStatus *apps.DaemonSetStatus) error {
if dsStatus.CurrentNumberScheduled != dsStatus.DesiredNumberScheduled {
return errors.Errorf("current number of scheduled Pods ('%d') doesn't match the amount of desired Pods ('%d')",
dsStatus.CurrentNumberScheduled, dsStatus.DesiredNumberScheduled)
}
if dsStatus.NumberAvailable == 0 {
return errors.New("no available Pods for DaemonSet")
}
if dsStatus.NumberReady == 0 {
return errors.New("no ready Pods for DaemonSet")
}
return nil
}
// getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady
func getNotReadyNodes(nodes []v1.Node) []string {
notReadyNodes := []string{}