Merge pull request #113583 from chendave/POC_resetCfg

kubeadm: implementation of `ResetConfiguration` API types
This commit is contained in:
Kubernetes Prow Robot 2023-07-14 04:05:48 -07:00 committed by GitHub
commit 95c8d61918
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 731 additions and 59 deletions

View File

@ -40,6 +40,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
fuzzNetworking,
fuzzJoinConfiguration,
fuzzJoinControlPlane,
fuzzResetConfiguration,
}
}
@ -133,3 +134,10 @@ func fuzzJoinConfiguration(obj *kubeadm.JoinConfiguration, c fuzz.Continue) {
func fuzzJoinControlPlane(obj *kubeadm.JoinControlPlane, c fuzz.Continue) {
c.FuzzNoCustom(obj)
}
func fuzzResetConfiguration(obj *kubeadm.ResetConfiguration, c fuzz.Continue) {
c.FuzzNoCustom(obj)
// Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail)
obj.CertificatesDir = "/tmp"
}

View File

@ -39,6 +39,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&InitConfiguration{},
&ClusterConfiguration{},
&JoinConfiguration{},
&ResetConfiguration{},
)
return nil
}

View File

@ -465,5 +465,35 @@ type ComponentConfig interface {
Get() interface{}
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ResetConfiguration contains a list of fields that are specifically "kubeadm reset"-only runtime information.
type ResetConfiguration struct {
metav1.TypeMeta
// CertificatesDir specifies the directory where the certificates are stored. If specified, it will be cleaned during the reset process.
CertificatesDir string
// CleanupTmpDir specifies whether the "/etc/kubernetes/tmp" directory should be cleaned during the reset process.
CleanupTmpDir bool
// CRISocket is used to retrieve container runtime info and used for the removal of the containers.
// If CRISocket is not specified by flag or config file, kubeadm will try to detect one valid CRISocket instead.
CRISocket string
// DryRun tells if the dry run mode is enabled, don't apply any change if it is and just output what would be done.
DryRun bool
// Force flag instructs kubeadm to reset the node without prompting for confirmation.
Force bool
// IgnorePreflightErrors provides a slice of pre-flight errors to be ignored during the reset process, e.g. 'IsPrivilegedUser,Swap'.
IgnorePreflightErrors []string
// SkipPhases is a list of phases to skip during command execution.
// The list of phases can be obtained with the "kubeadm reset phase --help" command.
SkipPhases []string
}
// ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig
type ComponentConfigMap map[string]ComponentConfig

View File

@ -198,3 +198,10 @@ func SetDefaults_NodeRegistration(obj *NodeRegistrationOptions) {
obj.ImagePullPolicy = DefaultImagePullPolicy
}
}
// SetDefaults_ResetConfiguration assigns default values for the ResetConfiguration object
func SetDefaults_ResetConfiguration(obj *ResetConfiguration) {
if obj.CertificatesDir == "" {
obj.CertificatesDir = DefaultCertificatesDir
}
}

View File

@ -27,6 +27,7 @@ limitations under the License.
// - TODO https://github.com/kubernetes/kubeadm/issues/2890
// - Support custom environment variables in control plane components under `ClusterConfiguration`.
// Use `APIServer.ExtraEnvs`, `ControllerManager.ExtraEnvs`, `Scheduler.ExtraEnvs`, `Etcd.Local.ExtraEnvs`.
// - The ResetConfiguration API type is now supported in v1beta4. Users are able to reset a node by passing a --config file to "kubeadm reset".
//
// Migration from old kubeadm config versions
//

View File

@ -48,6 +48,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&InitConfiguration{},
&ClusterConfiguration{},
&JoinConfiguration{},
&ResetConfiguration{},
)
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil

View File

@ -453,3 +453,40 @@ type Patches struct {
// +optional
Directory string `json:"directory,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ResetConfiguration contains a list of fields that are specifically "kubeadm reset"-only runtime information.
type ResetConfiguration struct {
metav1.TypeMeta `json:",inline"`
// CleanupTmpDir specifies whether the "/etc/kubernetes/tmp" directory should be cleaned during the reset process.
// +optional
CleanupTmpDir bool `json:"cleanupTmpDir,omitempty"`
// CertificatesDir specifies the directory where the certificates are stored. If specified, it will be cleaned during the reset process.
// +optional
CertificatesDir string `json:"certificatesDir,omitempty"`
// CRISocket is used to retrieve container runtime info and used for the removal of the containers.
// If CRISocket is not specified by flag or config file, kubeadm will try to detect one valid CRISocket instead.
// +optional
CRISocket string `json:"criSocket,omitempty"`
// DryRun tells if the dry run mode is enabled, don't apply any change if it is and just output what would be done.
// +optional
DryRun bool `json:"dryRun,omitempty"`
// Force flag instructs kubeadm to reset the node without prompting for confirmation.
// +optional
Force bool `json:"force,omitempty"`
// IgnorePreflightErrors provides a slice of pre-flight errors to be ignored during the reset process, e.g. 'IsPrivilegedUser,Swap'.
// +optional
IgnorePreflightErrors []string `json:"ignorePreflightErrors,omitempty"`
// SkipPhases is a list of phases to skip during command execution.
// The list of phases can be obtained with the "kubeadm reset phase --help" command.
// +optional
SkipPhases []string `json:"skipPhases,omitempty"`
}

View File

@ -219,6 +219,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*ResetConfiguration)(nil), (*kubeadm.ResetConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta4_ResetConfiguration_To_kubeadm_ResetConfiguration(a.(*ResetConfiguration), b.(*kubeadm.ResetConfiguration), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*kubeadm.ResetConfiguration)(nil), (*ResetConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kubeadm_ResetConfiguration_To_v1beta4_ResetConfiguration(a.(*kubeadm.ResetConfiguration), b.(*ResetConfiguration), scope)
}); err != nil {
return err
}
if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kubeadm_InitConfiguration_To_v1beta4_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope)
}); err != nil {
@ -769,3 +779,35 @@ func autoConvert_kubeadm_Patches_To_v1beta4_Patches(in *kubeadm.Patches, out *Pa
func Convert_kubeadm_Patches_To_v1beta4_Patches(in *kubeadm.Patches, out *Patches, s conversion.Scope) error {
return autoConvert_kubeadm_Patches_To_v1beta4_Patches(in, out, s)
}
func autoConvert_v1beta4_ResetConfiguration_To_kubeadm_ResetConfiguration(in *ResetConfiguration, out *kubeadm.ResetConfiguration, s conversion.Scope) error {
out.CleanupTmpDir = in.CleanupTmpDir
out.CertificatesDir = in.CertificatesDir
out.CRISocket = in.CRISocket
out.DryRun = in.DryRun
out.Force = in.Force
out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors))
out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
return nil
}
// Convert_v1beta4_ResetConfiguration_To_kubeadm_ResetConfiguration is an autogenerated conversion function.
func Convert_v1beta4_ResetConfiguration_To_kubeadm_ResetConfiguration(in *ResetConfiguration, out *kubeadm.ResetConfiguration, s conversion.Scope) error {
return autoConvert_v1beta4_ResetConfiguration_To_kubeadm_ResetConfiguration(in, out, s)
}
func autoConvert_kubeadm_ResetConfiguration_To_v1beta4_ResetConfiguration(in *kubeadm.ResetConfiguration, out *ResetConfiguration, s conversion.Scope) error {
out.CertificatesDir = in.CertificatesDir
out.CleanupTmpDir = in.CleanupTmpDir
out.CRISocket = in.CRISocket
out.DryRun = in.DryRun
out.Force = in.Force
out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors))
out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
return nil
}
// Convert_kubeadm_ResetConfiguration_To_v1beta4_ResetConfiguration is an autogenerated conversion function.
func Convert_kubeadm_ResetConfiguration_To_v1beta4_ResetConfiguration(in *kubeadm.ResetConfiguration, out *ResetConfiguration, s conversion.Scope) error {
return autoConvert_kubeadm_ResetConfiguration_To_v1beta4_ResetConfiguration(in, out, s)
}

View File

@ -518,3 +518,38 @@ func (in *Patches) DeepCopy() *Patches {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ResetConfiguration) DeepCopyInto(out *ResetConfiguration) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.IgnorePreflightErrors != nil {
in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.SkipPhases != nil {
in, out := &in.SkipPhases, &out.SkipPhases
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResetConfiguration.
func (in *ResetConfiguration) DeepCopy() *ResetConfiguration {
if in == nil {
return nil
}
out := new(ResetConfiguration)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ResetConfiguration) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}

View File

@ -33,6 +33,7 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
scheme.AddTypeDefaultingFunc(&ClusterConfiguration{}, func(obj interface{}) { SetObjectDefaults_ClusterConfiguration(obj.(*ClusterConfiguration)) })
scheme.AddTypeDefaultingFunc(&InitConfiguration{}, func(obj interface{}) { SetObjectDefaults_InitConfiguration(obj.(*InitConfiguration)) })
scheme.AddTypeDefaultingFunc(&JoinConfiguration{}, func(obj interface{}) { SetObjectDefaults_JoinConfiguration(obj.(*JoinConfiguration)) })
scheme.AddTypeDefaultingFunc(&ResetConfiguration{}, func(obj interface{}) { SetObjectDefaults_ResetConfiguration(obj.(*ResetConfiguration)) })
return nil
}
@ -91,3 +92,7 @@ func SetObjectDefaults_JoinConfiguration(in *JoinConfiguration) {
SetDefaults_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint)
}
}
func SetObjectDefaults_ResetConfiguration(in *ResetConfiguration) {
SetDefaults_ResetConfiguration(in)
}

View File

@ -655,3 +655,11 @@ func ValidateImageRepository(imageRepository string, fldPath *field.Path) field.
return allErrs
}
// ValidateResetConfiguration validates a ResetConfiguration object and collects all encountered errors
func ValidateResetConfiguration(c *kubeadm.ResetConfiguration) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, ValidateSocketPath(c.CRISocket, field.NewPath("criSocket"))...)
allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...)
return allErrs
}

View File

@ -1323,3 +1323,22 @@ func TestValidateImageRepository(t *testing.T) {
}
}
}
func TestValidateAbsolutePath(t *testing.T) {
var tests = []struct {
name string
path string
expectedErrors bool
}{
{name: "valid absolute path", path: "/etc/cert/dir", expectedErrors: false},
{name: "relative path", path: "./tmp", expectedErrors: true},
{name: "invalid path", path: "foo..", expectedErrors: true},
}
for _, tc := range tests {
actual := ValidateAbsolutePath(tc.path, field.NewPath("certificatesDir"))
actualErrors := len(actual) > 0
if actualErrors != tc.expectedErrors {
t.Errorf("error: validate absolute path: %q\n\texpected: %t\n\t actual: %t", tc.path, tc.expectedErrors, actualErrors)
}
}
}

View File

@ -548,3 +548,38 @@ func (in *Patches) DeepCopy() *Patches {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ResetConfiguration) DeepCopyInto(out *ResetConfiguration) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.IgnorePreflightErrors != nil {
in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.SkipPhases != nil {
in, out := &in.SkipPhases, &out.SkipPhases
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResetConfiguration.
func (in *ResetConfiguration) DeepCopy() *ResetConfiguration {
if in == nil {
return nil
}
out := new(ResetConfiguration)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ResetConfiguration) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}

View File

@ -32,6 +32,7 @@ type resetData interface {
InputReader() io.Reader
IgnorePreflightErrors() sets.Set[string]
Cfg() *kubeadmapi.InitConfiguration
ResetCfg() *kubeadmapi.ResetConfiguration
DryRun() bool
Client() clientset.Interface
CertificatesDir() string

View File

@ -31,12 +31,13 @@ type testData struct{}
// testData must satisfy resetData.
var _ resetData = &testData{}
func (t *testData) ForceReset() bool { return false }
func (t *testData) InputReader() io.Reader { return nil }
func (t *testData) IgnorePreflightErrors() sets.Set[string] { return nil }
func (t *testData) Cfg() *kubeadmapi.InitConfiguration { return nil }
func (t *testData) DryRun() bool { return false }
func (t *testData) Client() clientset.Interface { return nil }
func (t *testData) CertificatesDir() string { return "" }
func (t *testData) CRISocketPath() string { return "" }
func (t *testData) CleanupTmpDir() bool { return false }
func (t *testData) ForceReset() bool { return false }
func (t *testData) InputReader() io.Reader { return nil }
func (t *testData) IgnorePreflightErrors() sets.Set[string] { return nil }
func (t *testData) Cfg() *kubeadmapi.InitConfiguration { return nil }
func (t *testData) DryRun() bool { return false }
func (t *testData) Client() clientset.Interface { return nil }
func (t *testData) CertificatesDir() string { return "" }
func (t *testData) CRISocketPath() string { return "" }
func (t *testData) CleanupTmpDir() bool { return false }
func (t *testData) ResetCfg() *kubeadmapi.ResetConfiguration { return nil }

View File

@ -17,6 +17,7 @@ limitations under the License.
package cmd
import (
"errors"
"fmt"
"io"
"path"
@ -30,7 +31,9 @@ import (
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/reset"
@ -60,13 +63,10 @@ var (
// resetOptions defines all the options exposed via flags by kubeadm reset.
type resetOptions struct {
certificatesDir string
criSocketPath string
forceReset bool
ignorePreflightErrors []string
kubeconfigPath string
dryRun bool
cleanupTmpDir bool
cfgPath string
ignorePreflightErrors []string
externalcfg *v1beta4.ResetConfiguration
}
// resetData defines all the runtime information used when running the kubeadm reset workflow;
@ -80,93 +80,113 @@ type resetData struct {
inputReader io.Reader
outputWriter io.Writer
cfg *kubeadmapi.InitConfiguration
resetCfg *kubeadmapi.ResetConfiguration
dryRun bool
cleanupTmpDir bool
}
// newResetOptions returns a struct ready for being used for creating cmd join flags.
func newResetOptions() *resetOptions {
// initialize the public kubeadm config API by applying defaults
externalcfg := &v1beta4.ResetConfiguration{}
// Apply defaults
kubeadmscheme.Scheme.Default(externalcfg)
return &resetOptions{
certificatesDir: kubeadmapiv1.DefaultCertificatesDir,
forceReset: false,
kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
cleanupTmpDir: false,
kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
externalcfg: externalcfg,
}
}
// newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow.
func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) {
var cfg *kubeadmapi.InitConfiguration
func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.Writer, allowExperimental bool) (*resetData, error) {
// Validate the mixed arguments with --config and return early on errors
if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
return nil, err
}
client, err := cmdutil.GetClientSet(options.kubeconfigPath, false)
var initCfg *kubeadmapi.InitConfiguration
// Either use the config file if specified, or convert public kubeadm API to the internal ResetConfiguration and validates cfg.
resetCfg, err := configutil.LoadOrDefaultResetConfiguration(opts.cfgPath, opts.externalcfg, allowExperimental)
if err != nil {
return nil, err
}
client, err := cmdutil.GetClientSet(opts.kubeconfigPath, false)
if err == nil {
klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath)
cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", opts.kubeconfigPath)
initCfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
if err != nil {
klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err)
}
} else {
klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath)
klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", opts.kubeconfigPath)
}
ignorePreflightErrorsFromCfg := []string{}
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrorsFromCfg)
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opts.ignorePreflightErrors, resetCfg.IgnorePreflightErrors)
if err != nil {
return nil, err
}
if cfg != nil {
if initCfg != nil {
// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
cfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
initCfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
}
var criSocketPath string
if options.criSocketPath == "" {
criSocketPath, err = resetDetectCRISocket(cfg)
criSocketPath := opts.externalcfg.CRISocket
if criSocketPath == "" {
criSocketPath, err = resetDetectCRISocket(resetCfg, initCfg)
if err != nil {
return nil, err
}
klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath)
} else {
criSocketPath = options.criSocketPath
klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath)
}
certificatesDir := kubeadmapiv1.DefaultCertificatesDir
if cmd.Flags().Changed(options.CertificatesDir) { // flag is specified
certificatesDir = opts.externalcfg.CertificatesDir
} else if len(resetCfg.CertificatesDir) > 0 { // configured in the ResetConfiguration
certificatesDir = resetCfg.CertificatesDir
} else if len(initCfg.ClusterConfiguration.CertificatesDir) > 0 { // fetch from cluster
certificatesDir = initCfg.ClusterConfiguration.CertificatesDir
}
return &resetData{
certificatesDir: options.certificatesDir,
certificatesDir: certificatesDir,
client: client,
criSocketPath: criSocketPath,
forceReset: options.forceReset,
ignorePreflightErrors: ignorePreflightErrorsSet,
inputReader: in,
outputWriter: out,
cfg: cfg,
dryRun: options.dryRun,
cleanupTmpDir: options.cleanupTmpDir,
cfg: initCfg,
resetCfg: resetCfg,
dryRun: cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, resetCfg.DryRun, opts.externalcfg.DryRun).(bool),
forceReset: cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.ForceReset, resetCfg.Force, opts.externalcfg.Force).(bool),
cleanupTmpDir: cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CleanupTmpDir, resetCfg.CleanupTmpDir, opts.externalcfg.CleanupTmpDir).(bool),
}, nil
}
// AddResetFlags adds reset flags
func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
flagSet.StringVar(
&resetOptions.certificatesDir, options.CertificatesDir, resetOptions.certificatesDir,
&resetOptions.externalcfg.CertificatesDir, options.CertificatesDir, kubeadmapiv1.DefaultCertificatesDir,
`The path to the directory where the certificates are stored. If specified, clean this directory.`,
)
flagSet.BoolVarP(
&resetOptions.forceReset, options.ForceReset, "f", false,
&resetOptions.externalcfg.Force, options.ForceReset, "f", resetOptions.externalcfg.Force,
"Reset the node without prompting for confirmation.",
)
flagSet.BoolVar(
&resetOptions.dryRun, options.DryRun, resetOptions.dryRun,
&resetOptions.externalcfg.DryRun, options.DryRun, resetOptions.externalcfg.DryRun,
"Don't apply any changes; just output what would be done.",
)
flagSet.BoolVar(
&resetOptions.cleanupTmpDir, options.CleanupTmpDir, resetOptions.cleanupTmpDir,
&resetOptions.externalcfg.CleanupTmpDir, options.CleanupTmpDir, resetOptions.externalcfg.CleanupTmpDir,
fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
)
options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
options.AddConfigFlag(flagSet, &resetOptions.cfgPath)
options.AddIgnorePreflightErrorsFlag(flagSet, &resetOptions.ignorePreflightErrors)
cmdutil.AddCRISocketFlag(flagSet, &resetOptions.criSocketPath)
cmdutil.AddCRISocketFlag(flagSet, &resetOptions.externalcfg.CRISocket)
}
// newCmdReset returns the "kubeadm reset" command
@ -180,10 +200,16 @@ func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra
Use: "reset",
Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'",
RunE: func(cmd *cobra.Command, args []string) error {
err := resetRunner.Run(args)
data, err := resetRunner.InitData(args)
if err != nil {
return err
}
if _, ok := data.(*resetData); !ok {
return errors.New("invalid data struct")
}
if err := resetRunner.Run(args); err != nil {
return err
}
// output help text instructing user how to remove cni folders
fmt.Print(cniCleanupInstructions)
@ -194,7 +220,6 @@ func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra
}
AddResetFlags(cmd.Flags(), resetOptions)
// initialize the workflow runner with the list of phases
resetRunner.AppendPhase(phases.NewPreflightPhase())
resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
@ -206,9 +231,17 @@ func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra
if cmd.Flags().Lookup(options.NodeCRISocket) == nil {
// avoid CRI detection
// assume that the command execution does not depend on CRISocket when --cri-socket flag is not set
resetOptions.criSocketPath = kubeadmconstants.UnknownCRISocket
resetOptions.externalcfg.CRISocket = kubeadmconstants.UnknownCRISocket
}
return newResetData(cmd, resetOptions, in, out)
data, err := newResetData(cmd, resetOptions, in, out, true)
if err != nil {
return nil, err
}
// If the flag for skipping phases was empty, use the values from config
if len(resetRunner.Options.SkipPhases) == 0 {
resetRunner.Options.SkipPhases = data.resetCfg.SkipPhases
}
return data, nil
})
// binds the Runner to kubeadm reset command by altering
@ -218,6 +251,11 @@ func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra
return cmd
}
// ResetCfg returns the ResetConfiguration.
func (r *resetData) ResetCfg() *kubeadmapi.ResetConfiguration {
return r.resetCfg
}
// Cfg returns the InitConfiguration.
func (r *resetData) Cfg() *kubeadmapi.InitConfiguration {
return r.cfg
@ -263,12 +301,14 @@ func (r *resetData) CRISocketPath() string {
return r.criSocketPath
}
func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
if cfg != nil {
// first try to get the CRI socket from the cluster configuration
return cfg.NodeRegistration.CRISocket, nil
func resetDetectCRISocket(resetCfg *kubeadmapi.ResetConfiguration, initCfg *kubeadmapi.InitConfiguration) (string, error) {
if resetCfg != nil && len(resetCfg.CRISocket) > 0 {
return resetCfg.CRISocket, nil
}
if initCfg != nil && len(initCfg.NodeRegistration.CRISocket) > 0 {
return initCfg.NodeRegistration.CRISocket, nil
}
// if this fails, try to detect it
// try to detect it on host
return utilruntime.DetectCRISocket()
}

View File

@ -17,17 +17,54 @@ limitations under the License.
package cmd
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
var testResetConfig = fmt.Sprintf(`apiVersion: %s
kind: ResetConfiguration
force: true
dryRun: true
cleanupTmpDir: true
criSocket: unix:///var/run/fake.sock
certificatesDir: /etc/kubernetes/pki2
ignorePreflightErrors:
- a
- b
`, kubeadmapiv1.SchemeGroupVersion.String())
func TestNewResetData(t *testing.T) {
// create temp directory
tmpDir, err := os.MkdirTemp("", "kubeadm-reset-test")
if err != nil {
t.Errorf("Unable to create temporary directory: %v", err)
}
defer os.RemoveAll(tmpDir)
// create config file
configFilePath := filepath.Join(tmpDir, "test-config-file")
cfgFile, err := os.Create(configFilePath)
if err != nil {
t.Errorf("Unable to create file %q: %v", configFilePath, err)
}
defer cfgFile.Close()
if _, err = cfgFile.WriteString(testResetConfig); err != nil {
t.Fatalf("Unable to write file %q: %v", configFilePath, err)
}
testCases := []struct {
name string
args []string
@ -40,7 +77,7 @@ func TestNewResetData(t *testing.T) {
name: "flags parsed correctly",
flags: map[string]string{
options.CertificatesDir: "/tmp",
options.NodeCRISocket: "unix:///var/run/crio/crio.sock",
options.NodeCRISocket: constants.CRISocketCRIO,
options.IgnorePreflightErrors: "all",
options.ForceReset: "true",
options.DryRun: "true",
@ -48,11 +85,20 @@ func TestNewResetData(t *testing.T) {
},
data: &resetData{
certificatesDir: "/tmp",
criSocketPath: "unix:///var/run/crio/crio.sock",
criSocketPath: constants.CRISocketCRIO,
ignorePreflightErrors: sets.New("all"),
forceReset: true,
dryRun: true,
cleanupTmpDir: true,
// resetCfg holds the value passed from flags except the value of ignorePreflightErrors
resetCfg: &kubeadmapi.ResetConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "", APIVersion: ""},
Force: true,
CertificatesDir: "/tmp",
CRISocket: constants.CRISocketCRIO,
DryRun: true,
CleanupTmpDir: true,
},
},
},
{
@ -71,6 +117,100 @@ func TestNewResetData(t *testing.T) {
},
validate: expectedResetIgnorePreflightErrors(sets.New("a", "b")),
},
// Start the testcases with config file
{
name: "Pass with config from file",
flags: map[string]string{
options.CfgPath: configFilePath,
},
data: &resetData{
certificatesDir: "/etc/kubernetes/pki2", // cover the case that default is overridden as well
criSocketPath: "unix:///var/run/fake.sock", // cover the case that default is overridden as well
ignorePreflightErrors: sets.New("a", "b"),
forceReset: true,
dryRun: true,
cleanupTmpDir: true,
resetCfg: &kubeadmapi.ResetConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "", APIVersion: ""},
Force: true,
CertificatesDir: "/etc/kubernetes/pki2",
CRISocket: "unix:///var/run/fake.sock",
IgnorePreflightErrors: []string{"a", "b"},
CleanupTmpDir: true,
DryRun: true,
},
},
},
{
name: "force from config file overrides default",
flags: map[string]string{
options.CfgPath: configFilePath,
},
validate: func(t *testing.T, data *resetData) {
// validate that the default value is overwritten
if data.forceReset != true {
t.Error("Invalid forceReset")
}
},
},
{
name: "dryRun configured in the config file only",
flags: map[string]string{
options.CfgPath: configFilePath,
},
validate: func(t *testing.T, data *resetData) {
if data.dryRun != true {
t.Error("Invalid dryRun")
}
},
},
{
name: "--cert-dir flag is not allowed to mix with config",
flags: map[string]string{
options.CfgPath: configFilePath,
options.CertificatesDir: "/tmp",
},
expectError: "can not mix '--config' with arguments",
},
{
name: "--cri-socket flag is not allowed to mix with config",
flags: map[string]string{
options.CfgPath: configFilePath,
options.NodeCRISocket: "unix:///var/run/bogus.sock",
},
expectError: "can not mix '--config' with arguments",
},
{
name: "--force flag is not allowed to mix with config",
flags: map[string]string{
options.CfgPath: configFilePath,
options.ForceReset: "false",
},
expectError: "can not mix '--config' with arguments",
},
{
name: "--cleanup-tmp-dir flag is not allowed to mix with config",
flags: map[string]string{
options.CfgPath: configFilePath,
options.CleanupTmpDir: "true",
},
expectError: "can not mix '--config' with arguments",
},
{
name: "pre-flights errors from ResetConfiguration only",
flags: map[string]string{
options.CfgPath: configFilePath,
},
validate: expectedResetIgnorePreflightErrors(sets.New("a", "b")),
},
{
name: "pre-flights errors from both CLI args and ResetConfiguration",
flags: map[string]string{
options.CfgPath: configFilePath,
options.IgnorePreflightErrors: "c,d",
},
validate: expectedResetIgnorePreflightErrors(sets.New("a", "b", "c", "d")),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
@ -84,7 +224,7 @@ func TestNewResetData(t *testing.T) {
}
// test newResetData method
data, err := newResetData(cmd, resetOptions, nil, nil)
data, err := newResetData(cmd, resetOptions, nil, nil, true)
if err != nil && !strings.Contains(err.Error(), tc.expectError) {
t.Fatalf("newResetData returned unexpected error, expected: %s, got %v", tc.expectError, err)
}

View File

@ -141,3 +141,12 @@ func GetClientSet(file string, dryRun bool) (clientset.Interface, error) {
}
return kubeconfigutil.ClientSetFromFile(file)
}
// ValueFromFlagsOrConfig checks if the "name" flag has been set. If yes, it returns the value of the flag, otherwise it returns the value from config.
func ValueFromFlagsOrConfig(flagSet *pflag.FlagSet, name string, cfgValue interface{}, flagValue interface{}) interface{} {
if flagSet.Changed(name) {
return flagValue
}
// assume config has all the defaults set correctly.
return cfgValue
}

View File

@ -352,6 +352,9 @@ const (
// JoinConfigurationKind is the string kind value for the JoinConfiguration struct
JoinConfigurationKind = "JoinConfiguration"
// ResetConfigurationKind is the string kind value for the ResetConfiguration struct
ResetConfigurationKind = "ResetConfiguration"
// YAMLDocumentSeparator is the separator for YAML documents
// TODO: Find a better place for this constant
YAMLDocumentSeparator = "---\n"

View File

@ -91,7 +91,7 @@ func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated, allowExpe
}
if _, present := deprecatedAPIVersions[gvString]; present && !allowDeprecated {
klog.Warningf("your configuration file uses a deprecated API spec: %q. Please use 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv)
klog.Warningf("your configuration file uses a deprecated API spec: %q. Please use 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv.String())
}
if _, present := experimentalAPIVersions[gvString]; present && !allowExperimental {

View File

@ -0,0 +1,154 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"os"
"strings"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict"
kubeadmruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
)
// SetResetDynamicDefaults checks and sets configuration values for the ResetConfiguration object
func SetResetDynamicDefaults(cfg *kubeadmapi.ResetConfiguration) error {
var err error
if cfg.CRISocket == "" {
cfg.CRISocket, err = kubeadmruntime.DetectCRISocket()
if err != nil {
return err
}
klog.V(1).Infof("detected and using CRI socket: %s", cfg.CRISocket)
} else {
if !strings.HasPrefix(cfg.CRISocket, kubeadmapiv1.DefaultContainerRuntimeURLScheme) {
klog.Warningf("Usage of CRI endpoints without URL scheme is deprecated and can cause kubelet errors "+
"in the future. Automatically prepending scheme %q to the \"criSocket\" with value %q. "+
"Please update your configuration!", kubeadmapiv1.DefaultContainerRuntimeURLScheme, cfg.CRISocket)
cfg.CRISocket = kubeadmapiv1.DefaultContainerRuntimeURLScheme + "://" + cfg.CRISocket
}
}
return nil
}
// LoadOrDefaultResetConfiguration takes a path to a config file and a versioned configuration that can serve as the default config
// If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used.
// Then the external, versioned configuration is defaulted and converted to the internal type.
// Right thereafter, the configuration is defaulted again with dynamic values
// Lastly, the internal config is validated and returned.
func LoadOrDefaultResetConfiguration(cfgPath string, defaultversionedcfg *kubeadmapiv1.ResetConfiguration, allowExperimental bool) (*kubeadmapi.ResetConfiguration, error) {
if cfgPath != "" {
// Loads configuration from config file, if provided
return LoadResetConfigurationFromFile(cfgPath, allowExperimental)
}
return DefaultedResetConfiguration(defaultversionedcfg)
}
// LoadResetConfigurationFromFile loads versioned ResetConfiguration from file, converts it to internal, defaults and validates it
func LoadResetConfigurationFromFile(cfgPath string, allowExperimental bool) (*kubeadmapi.ResetConfiguration, error) {
klog.V(1).Infof("loading configuration from %q", cfgPath)
b, err := os.ReadFile(cfgPath)
if err != nil {
return nil, errors.Wrapf(err, "unable to read config from %q ", cfgPath)
}
gvkmap, err := kubeadmutil.SplitYAMLDocuments(b)
if err != nil {
return nil, err
}
return documentMapToResetConfiguration(gvkmap, false, allowExperimental)
}
// documentMapToResetConfiguration takes a map between GVKs and YAML documents (as returned by SplitYAMLDocuments),
// finds a ResetConfiguration, decodes it, dynamically defaults it and then validates it prior to return.
func documentMapToResetConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated, allowExperimental bool) (*kubeadmapi.ResetConfiguration, error) {
resetBytes := []byte{}
for gvk, bytes := range gvkmap {
// not interested in anything other than ResetConfiguration
if gvk.Kind != constants.ResetConfigurationKind {
continue
}
// check if this version is supported and possibly not deprecated
if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated, allowExperimental); err != nil {
return nil, err
}
// verify the validity of the YAML
if err := strict.VerifyUnmarshalStrict([]*runtime.Scheme{kubeadmscheme.Scheme}, gvk, bytes); err != nil {
klog.Warning(err.Error())
}
resetBytes = bytes
}
if len(resetBytes) == 0 {
return nil, errors.Errorf("no %s found in the supplied config", constants.JoinConfigurationKind)
}
internalcfg := &kubeadmapi.ResetConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), resetBytes, internalcfg); err != nil {
return nil, err
}
// Applies dynamic defaults to settings not provided with flags
if err := SetResetDynamicDefaults(internalcfg); err != nil {
return nil, err
}
// Validates cfg
if err := validation.ValidateResetConfiguration(internalcfg).ToAggregate(); err != nil {
return nil, err
}
return internalcfg, nil
}
// DefaultedResetConfiguration takes a versioned ResetConfiguration (usually filled in by command line parameters), defaults it, converts it to internal and validates it
func DefaultedResetConfiguration(defaultversionedcfg *kubeadmapiv1.ResetConfiguration) (*kubeadmapi.ResetConfiguration, error) {
internalcfg := &kubeadmapi.ResetConfiguration{}
// Takes passed flags into account; the defaulting is executed once again enforcing assignment of
// static default values to cfg only for values not provided with flags
kubeadmscheme.Scheme.Default(defaultversionedcfg)
if err := kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil); err != nil {
return nil, err
}
// Applies dynamic defaults to settings not provided with flags
if err := SetResetDynamicDefaults(internalcfg); err != nil {
return nil, err
}
// Validates cfg
if err := validation.ValidateResetConfiguration(internalcfg).ToAggregate(); err != nil {
return nil, err
}
return internalcfg, nil
}

View File

@ -0,0 +1,95 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"os"
"path/filepath"
"testing"
"github.com/lithammer/dedent"
)
func TestLoadResetConfigurationFromFile(t *testing.T) {
// Create temp folder for the test case
tmpdir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir: %v", err)
}
defer os.RemoveAll(tmpdir)
var tests = []struct {
name string
fileContents string
expectErr bool
}{
{
name: "empty file causes error",
expectErr: true,
},
{
name: "Invalid v1beta4 causes error",
fileContents: dedent.Dedent(`
apiVersion: kubeadm.k8s.io/unknownVersion
kind: ResetConfiguration
criSocket: unix:///var/run/containerd/containerd.sock
`),
expectErr: true,
},
{
name: "valid v1beta4 is loaded",
fileContents: dedent.Dedent(`
apiVersion: kubeadm.k8s.io/v1beta4
kind: ResetConfiguration
force: true
cleanupTmpDir: true
criSocket: unix:///var/run/containerd/containerd.sock
certificatesDir: /etc/kubernetes/pki
ignorePreflightErrors:
- a
- b
`),
},
}
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
cfgPath := filepath.Join(tmpdir, rt.name)
err := os.WriteFile(cfgPath, []byte(rt.fileContents), 0644)
if err != nil {
t.Errorf("Couldn't create file: %v", err)
return
}
obj, err := LoadResetConfigurationFromFile(cfgPath, true)
if rt.expectErr {
if err == nil {
t.Error("Unexpected success")
}
} else {
if err != nil {
t.Errorf("Error reading file: %v", err)
return
}
if obj == nil {
t.Error("Unexpected nil return value")
}
}
})
}
}