From f9f91bf18e639a8ecee12391d0fb149539e9a8f8 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Sat, 5 Aug 2017 16:44:39 +0200 Subject: [PATCH] fully implement kubeadm-phase-kubeconfig --- cmd/kubeadm/BUILD | 2 +- cmd/kubeadm/app/apis/kubeadm/types.go | 5 + cmd/kubeadm/app/cmd/init.go | 4 +- cmd/kubeadm/app/cmd/phases/BUILD | 13 +- cmd/kubeadm/app/cmd/phases/certs.go | 13 +- cmd/kubeadm/app/cmd/phases/certs_test.go | 156 ++----- cmd/kubeadm/app/cmd/phases/kubeconfig.go | 191 +++++--- cmd/kubeadm/app/cmd/phases/kubeconfig_test.go | 383 +++++++++++++++ cmd/kubeadm/app/phases/kubeconfig/BUILD | 19 + .../app/phases/kubeconfig/kubeconfig.go | 335 +++++++++----- .../app/phases/kubeconfig/kubeconfig_test.go | 435 ++++++++++++++++++ cmd/kubeadm/app/util/kubeconfig/kubeconfig.go | 1 - cmd/kubeadm/test/BUILD | 39 ++ cmd/kubeadm/test/certs/BUILD | 28 ++ cmd/kubeadm/test/certs/util.go | 79 ++++ cmd/kubeadm/test/cmd/BUILD | 1 + cmd/kubeadm/test/cmd/util.go | 34 ++ cmd/kubeadm/test/kubeconfig/BUILD | 31 ++ cmd/kubeadm/test/kubeconfig/util.go | 100 ++++ cmd/kubeadm/test/util.go | 126 +++++ 20 files changed, 1686 insertions(+), 309 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/kubeconfig_test.go create mode 100644 cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go create mode 100644 cmd/kubeadm/test/BUILD create mode 100644 cmd/kubeadm/test/certs/BUILD create mode 100644 cmd/kubeadm/test/certs/util.go create mode 100644 cmd/kubeadm/test/kubeconfig/BUILD create mode 100644 cmd/kubeadm/test/kubeconfig/util.go create mode 100644 cmd/kubeadm/test/util.go diff --git a/cmd/kubeadm/BUILD b/cmd/kubeadm/BUILD index f15bf064725..c15300af93b 100644 --- a/cmd/kubeadm/BUILD +++ b/cmd/kubeadm/BUILD @@ -41,7 +41,7 @@ filegroup( srcs = [ ":package-srcs", "//cmd/kubeadm/app:all-srcs", - "//cmd/kubeadm/test/cmd:all-srcs", + "//cmd/kubeadm/test:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 8968ff77742..2e132f913cd 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -17,6 +17,7 @@ limitations under the License. package kubeadm import ( + "fmt" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -105,3 +106,7 @@ type NodeConfiguration struct { TLSBootstrapToken string Token string } + +func (cfg *MasterConfiguration) GetMasterEndpoint() string { + return fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort) +} diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ee49701efd8..bd2e8266663 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -231,9 +231,7 @@ func (i *Init) Run(out io.Writer) error { } // PHASE 2: Generate kubeconfig files for the admin and the kubelet - - masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort) - err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmconstants.KubernetesDir, i.cfg.NodeName) + err = kubeconfigphase.CreateInitKubeConfigFiles(kubeadmconstants.KubernetesDir, i.cfg) if err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index abb2e2d0638..5975ef330dc 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -40,15 +40,22 @@ go_library( go_test( name = "go_default_test", - srcs = ["certs_test.go"], + srcs = [ + "certs_test.go", + "kubeconfig_test.go", + ], library = ":go_default_library", tags = ["automanaged"], deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/install:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", - "//vendor/github.com/renstrom/dedent:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", + "//cmd/kubeadm/test:go_default_library", + "//cmd/kubeadm/test/cmd:go_default_library", + "//cmd/kubeadm/test/kubeconfig:go_default_library", + "//pkg/util/node:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index e1470131197..a06dd885853 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/api" ) +// NewCmdCerts return main command for certs phase func NewCmdCerts() *cobra.Command { cmd := &cobra.Command{ Use: "certs", @@ -42,12 +43,12 @@ func NewCmdCerts() *cobra.Command { RunE: subCmdRunE("certs"), } - cmd.AddCommand(newSubCmdCerts()...) + cmd.AddCommand(getCertsSubCommands()...) return cmd } -// newSubCmdCerts returns sub commands for certs phase -func newSubCmdCerts() []*cobra.Command { +// getCertsSubCommands returns sub commands for certs phase +func getCertsSubCommands() []*cobra.Command { cfg := &kubeadmapiext.MasterConfiguration{} // Default values for the cobra help text @@ -122,13 +123,13 @@ func newSubCmdCerts() []*cobra.Command { return subCmds } -// runCmdFunc creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of inpunt parameters) +// runCmdFunc creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of input parameters) func runCmdFunc(cmdFunc func(cfg *kubeadmapi.MasterConfiguration) error, cfgPath *string, cfg *kubeadmapiext.MasterConfiguration) func(cmd *cobra.Command, args []string) { - // the following statement build a clousure that wraps a call to a CreateCertFunc, binding + // the following statement build a clousure that wraps a call to a cmdFunc, binding // the function itself with the specific parameters of each sub command. // Please note that specific parameter should be passed as value, while other parameters - passed as reference - - // are shared between sub commnands and gets access to current value e.g. flags value. + // are shared between sub commands and gets access to current value e.g. flags value. return func(cmd *cobra.Command, args []string) { internalcfg := &kubeadmapi.MasterConfiguration{} diff --git a/cmd/kubeadm/app/cmd/phases/certs_test.go b/cmd/kubeadm/app/cmd/phases/certs_test.go index 6a50c86acd1..8cb5e7414e2 100644 --- a/cmd/kubeadm/app/cmd/phases/certs_test.go +++ b/cmd/kubeadm/app/cmd/phases/certs_test.go @@ -18,26 +18,24 @@ package phases import ( "fmt" - "html/template" - "io/ioutil" "os" - "path" - "strings" "testing" - "github.com/renstrom/dedent" - "github.com/spf13/cobra" - // required for triggering api machinery startup when running unit tests _ "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/install" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + "k8s.io/kubernetes/pkg/util/node" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" + cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" ) func TestSubCmdCertsCreateFiles(t *testing.T) { - subCmds := newSubCmdCerts() + subCmds := getCertsSubCommands() var tests = []struct { subCmds []string @@ -81,71 +79,51 @@ func TestSubCmdCertsCreateFiles(t *testing.T) { } for _, test := range tests { - // Temporary folder for the test case - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) defer os.RemoveAll(tmpdir) // executes given sub commands for _, subCmdName := range test.subCmds { - subCmd := getSubCmd(t, subCmdName, subCmds) - subCmd.SetArgs([]string{fmt.Sprintf("--cert-dir=%s", tmpdir)}) - if err := subCmd.Execute(); err != nil { - t.Fatalf("Could not execute subcommand: %s", subCmdName) - } + certDirFlag := fmt.Sprintf("--cert-dir=%s", tmpdir) + cmdtestutil.RunSubCommand(t, subCmds, subCmdName, certDirFlag) } // verify expected files are there - assertFilesCount(t, tmpdir, len(test.expectedFiles)) - for _, file := range test.expectedFiles { - assertFileExists(t, tmpdir, file) - } + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) } } func TestSubCmdApiServerFlags(t *testing.T) { - subCmds := newSubCmdCerts() + subCmds := getCertsSubCommands() - // Temporary folder for the test case - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) defer os.RemoveAll(tmpdir) // creates ca cert - subCmd := getSubCmd(t, "ca", subCmds) - subCmd.SetArgs([]string{fmt.Sprintf("--cert-dir=%s", tmpdir)}) - if err := subCmd.Execute(); err != nil { - t.Fatalf("Could not execute subcommand ca") - } + certDirFlag := fmt.Sprintf("--cert-dir=%s", tmpdir) + cmdtestutil.RunSubCommand(t, subCmds, "ca", certDirFlag) // creates apiserver cert - subCmd = getSubCmd(t, "apiserver", subCmds) - subCmd.SetArgs([]string{ + apiserverFlags := []string{ fmt.Sprintf("--cert-dir=%s", tmpdir), "--apiserver-cert-extra-sans=foo,boo", "--service-cidr=10.0.0.0/24", "--service-dns-domain=mycluster.local", "--apiserver-advertise-address=1.2.3.4", - }) - if err := subCmd.Execute(); err != nil { - t.Fatalf("Could not execute subcommand apiserver") } + cmdtestutil.RunSubCommand(t, subCmds, "apiserver", apiserverFlags...) APIserverCert, err := pkiutil.TryLoadCertFromDisk(tmpdir, kubeadmconstants.APIServerCertAndKeyBaseName) if err != nil { t.Fatalf("Error loading API server certificate: %v", err) } - hostname, err := os.Hostname() - if err != nil { - t.Errorf("couldn't get the hostname: %v", err) - } - for i, name := range []string{strings.ToLower(hostname), "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.mycluster.local"} { + hostname := node.GetHostname("") + + for i, name := range []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.mycluster.local"} { if APIserverCert.DNSNames[i] != name { t.Errorf("APIserverCert.DNSNames[%d] is %s instead of %s", i, APIserverCert.DNSNames[i], name) } @@ -157,9 +135,9 @@ func TestSubCmdApiServerFlags(t *testing.T) { } } -func TestSubCmdReadsConfig(t *testing.T) { +func TestSubCmdCertsReadsConfig(t *testing.T) { - subCmds := newSubCmdCerts() + subCmds := getCertsSubCommands() var tests = []struct { subCmds []string @@ -184,88 +162,26 @@ func TestSubCmdReadsConfig(t *testing.T) { } for _, test := range tests { - // Temporary folder for the test case - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) defer os.RemoveAll(tmpdir) - configPath := saveDummyCfg(t, tmpdir) + certdir := tmpdir + + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + CertificatesDir: certdir, + NodeName: "valid-node-name", + } + configPath := testutil.SetupMasterConfigurationFile(t, tmpdir, cfg) // executes given sub commands for _, subCmdName := range test.subCmds { - subCmd := getSubCmd(t, subCmdName, subCmds) - subCmd.SetArgs([]string{fmt.Sprintf("--config=%s", configPath)}) - if err := subCmd.Execute(); err != nil { - t.Fatalf("Could not execute command: %s", subCmdName) - } + configFlag := fmt.Sprintf("--config=%s", configPath) + cmdtestutil.RunSubCommand(t, subCmds, subCmdName, configFlag) } // verify expected files are there - // NB. test.expectedFileCount + 1 because in this test case the tempdir where key/certificates - // are saved contains also the dummy configuration file - assertFilesCount(t, tmpdir, test.expectedFileCount+1) + testutil.AssertFilesCount(t, tmpdir, test.expectedFileCount) } } - -func getSubCmd(t *testing.T, name string, subCmds []*cobra.Command) *cobra.Command { - for _, subCmd := range subCmds { - if subCmd.Name() == name { - return subCmd - } - } - t.Fatalf("Unable to find sub command %s", name) - - return nil -} - -func assertFilesCount(t *testing.T, dirName string, count int) { - files, err := ioutil.ReadDir(dirName) - if err != nil { - t.Fatalf("Couldn't read files from tmpdir: %s", err) - } - - if len(files) != count { - t.Errorf("dir does contains %d, %d expected", len(files), count) - for _, f := range files { - t.Error(f.Name()) - } - } -} - -func assertFileExists(t *testing.T, dirName string, fileName string) { - path := path.Join(dirName, fileName) - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Errorf("file %s does not exist", fileName) - } -} - -func saveDummyCfg(t *testing.T, dirName string) string { - - path := path.Join(dirName, "dummyconfig.yaml") - cfgTemplate := template.Must(template.New("init").Parse(dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1alpha1 - kind: MasterConfiguration - certificatesDir: {{.CertificatesDir}} - `))) - - f, err := os.Create(path) - if err != nil { - t.Errorf("error creating dummyconfig file %s: %v", path, err) - } - - templateData := struct { - CertificatesDir string - }{ - CertificatesDir: dirName, - } - - err = cfgTemplate.Execute(f, templateData) - if err != nil { - t.Errorf("error generating dummyconfig file %s: %v", path, err) - } - f.Close() - - return path -} diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index 998e4bfdc7b..a009a5cb998 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -22,98 +22,147 @@ import ( "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/pkg/api" ) +// NewCmdKubeConfig return main command for kubeconfig phase func NewCmdKubeConfig(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "kubeconfig", - Short: "Create KubeConfig files from given credentials.", + Short: "Generate all kubeconfig files necessary to establish the control plane and the admin kubeconfig file.", RunE: subCmdRunE("kubeconfig"), } - cmd.AddCommand(NewCmdToken(out)) - cmd.AddCommand(NewCmdClientCerts(out)) + cmd.AddCommand(getKubeConfigSubCommands(out, kubeadmconstants.KubernetesDir)...) return cmd } -func NewCmdToken(out io.Writer) *cobra.Command { - config := &kubeconfigphase.BuildConfigProperties{ - MakeClientCerts: false, - } - cmd := &cobra.Command{ - Use: "token", - Short: "Output a valid KubeConfig file to STDOUT with a token as the authentication method.", - Run: func(cmd *cobra.Command, args []string) { - err := RunCreateWithToken(out, config) - kubeadmutil.CheckErr(err) +// getKubeConfigSubCommands returns sub commands for kubeconfig phase +func getKubeConfigSubCommands(out io.Writer, outDir string) []*cobra.Command { + + cfg := &kubeadmapiext.MasterConfiguration{} + // Default values for the cobra help text + api.Scheme.Default(cfg) + + var cfgPath, token, clientName string + var subCmds []*cobra.Command + + subCmdProperties := []struct { + use string + short string + cmdFunc func(outDir string, cfg *kubeadmapi.MasterConfiguration) error + }{ + { + use: "all", + short: "Generate all kubeconfig files necessary to establish the control plane and the admin kubeconfig file.", + cmdFunc: kubeconfigphase.CreateInitKubeConfigFiles, + }, + { + use: "admin", + short: "Generate a kubeconfig file for the admin to use and for kubeadm itself.", + cmdFunc: kubeconfigphase.CreateAdminKubeConfigFile, + }, + { + use: "kubelet", + short: "Generate a kubeconfig file for the Kubelet to use. Please note that this should *only* be used for bootstrapping purposes. After your control plane is up, you should request all kubelet credentials from the CSR API.", + cmdFunc: kubeconfigphase.CreateKubeletKubeConfigFile, + }, + { + use: "controller-manager", + short: "Generate a kubeconfig file for the Controller Manager to use.", + cmdFunc: kubeconfigphase.CreateControllerManagerKubeConfigFile, + }, + { + use: "scheduler", + short: "Generate a kubeconfig file for the Scheduler to use.", + cmdFunc: kubeconfigphase.CreateSchedulerKubeConfigFile, + }, + { + use: "user", + short: "Outputs a kubeconfig file for an additional user.", + cmdFunc: func(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + if clientName == "" { + return fmt.Errorf("missing required argument client-name") + } + + // if the kubeconfig file for an additional user has to use a token, use it + if token != "" { + return kubeconfigphase.WriteKubeConfigWithToken(out, cfg, clientName, token) + } + + // Otherwise, write a kubeconfig file with a generate client cert + return kubeconfigphase.WriteKubeConfigWithClientCert(out, cfg, clientName) + }, }, } - addCommonFlags(cmd, config) - cmd.Flags().StringVar(&config.Token, "token", "", "The path to the directory where the certificates are.") - return cmd + + for _, properties := range subCmdProperties { + // Creates the UX Command + cmd := &cobra.Command{ + Use: properties.use, + Short: properties.short, + Run: runCmdFuncKubeConfig(properties.cmdFunc, &outDir, &cfgPath, cfg), + } + + // Add flags to the command + if properties.use != "user" { + cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + } + cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, "The path where to save and store the certificates") + cmd.Flags().StringVar(&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress, "The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.") + cmd.Flags().Int32Var(&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort, "Port for the API Server to bind to") + if properties.use == "all" || properties.use == "kubelet" { + cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, `Specify the node name`) + } + if properties.use == "user" { + cmd.Flags().StringVar(&token, "token", token, "The path to the directory where the certificates are.") + cmd.Flags().StringVar(&clientName, "client-name", clientName, "The name of the client for which the KubeConfig file will be generated.") + } + + subCmds = append(subCmds, cmd) + } + + return subCmds } -func NewCmdClientCerts(out io.Writer) *cobra.Command { - config := &kubeconfigphase.BuildConfigProperties{ - MakeClientCerts: true, - } - cmd := &cobra.Command{ - Use: "client-certs", - Short: "Output a valid KubeConfig file to STDOUT with a client certificates as the authentication method.", - Run: func(cmd *cobra.Command, args []string) { - err := RunCreateWithClientCerts(out, config) - kubeadmutil.CheckErr(err) - }, - } - addCommonFlags(cmd, config) - cmd.Flags().StringSliceVar(&config.Organization, "organization", []string{}, "The organization (group) the certificate should be in.") - return cmd -} +// runCmdFuncKubeConfig creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of input parameters) +func runCmdFuncKubeConfig(cmdFunc func(outDir string, cfg *kubeadmapi.MasterConfiguration) error, outDir, cfgPath *string, cfg *kubeadmapiext.MasterConfiguration) func(cmd *cobra.Command, args []string) { -func addCommonFlags(cmd *cobra.Command, config *kubeconfigphase.BuildConfigProperties) { - cmd.Flags().StringVar(&config.CertDir, "cert-dir", kubeadmapiext.DefaultCertificatesDir, "The path to the directory where the certificates are.") - cmd.Flags().StringVar(&config.ClientName, "client-name", "", "The name of the client for which the KubeConfig file will be generated.") - cmd.Flags().StringVar(&config.APIServer, "server", "", "The location of the api server.") -} + // the following statement build a clousure that wraps a call to a CreateKubeConfigFunc, binding + // the function itself with the specific parameters of each sub command. + // Please note that specific parameter should be passed as value, while other parameters - passed as reference - + // are shared between sub commands and gets access to current value e.g. flags value. -func validateCommonFlags(config *kubeconfigphase.BuildConfigProperties) error { - if len(config.ClientName) == 0 { - return fmt.Errorf("The --client-name flag is required") - } - if len(config.APIServer) == 0 { - return fmt.Errorf("The --server flag is required") - } - return nil -} + return func(cmd *cobra.Command, args []string) { + internalcfg := &kubeadmapi.MasterConfiguration{} -// RunCreateWithToken generates a kubeconfig file from with a token as the authentication mechanism -func RunCreateWithToken(out io.Writer, config *kubeconfigphase.BuildConfigProperties) error { - if len(config.Token) == 0 { - return fmt.Errorf("The --token flag is required") - } - if err := validateCommonFlags(config); err != nil { - return err - } - kubeConfigBytes, err := kubeconfigphase.GetKubeConfigBytesFromSpec(*config) - if err != nil { - return err - } - fmt.Fprintln(out, string(kubeConfigBytes)) - return nil -} + // Takes passed flags into account; the defaulting is executed once again enforcing assignement of + // static default values to cfg only for values not provided with flags + api.Scheme.Default(cfg) + api.Scheme.Convert(cfg, internalcfg, nil) -// RunCreateWithClientCerts generates a kubeconfig file from with client certs as the authentication mechanism -func RunCreateWithClientCerts(out io.Writer, config *kubeconfigphase.BuildConfigProperties) error { - if err := validateCommonFlags(config); err != nil { - return err + // Loads configuration from config file, if provided + // Nb. --config overrides command line flags + err := configutil.TryLoadMasterConfiguration(*cfgPath, internalcfg) + kubeadmutil.CheckErr(err) + + // Applies dynamic defaults to settings not provided with flags + err = configutil.SetInitDynamicDefaults(internalcfg) + kubeadmutil.CheckErr(err) + + // Validates cfg (flags/configs + defaults + dynamic defaults) + err = validation.ValidateMasterConfiguration(internalcfg).ToAggregate() + kubeadmutil.CheckErr(err) + + // Execute the cmdFunc + err = cmdFunc(*outDir, internalcfg) + kubeadmutil.CheckErr(err) } - kubeConfigBytes, err := kubeconfigphase.GetKubeConfigBytesFromSpec(*config) - if err != nil { - return err - } - fmt.Fprintln(out, string(kubeConfigBytes)) - return nil } diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig_test.go b/cmd/kubeadm/app/cmd/phases/kubeconfig_test.go new file mode 100644 index 00000000000..630ba6745a2 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig_test.go @@ -0,0 +1,383 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "testing" + + // required for triggering api machinery startup when running unit tests + _ "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/install" + + "k8s.io/client-go/tools/clientcmd" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" + cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" + kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig" +) + +func TestKubeConfigCSubCommandsHasFlags(t *testing.T) { + + subCmds := getKubeConfigSubCommands(nil, "") + + commonFlags := []string{ + "cert-dir", + "apiserver-advertise-address", + "apiserver-bind-port", + } + + var tests = []struct { + command string + additionalFlags []string + }{ + { + command: "all", + additionalFlags: []string{ + "config", + "node-name", + }, + }, + { + command: "admin", + additionalFlags: []string{ + "config", + }, + }, + { + command: "kubelet", + additionalFlags: []string{ + "config", + "node-name", + }, + }, + { + command: "controller-manager", + additionalFlags: []string{ + "config", + }, + }, + { + command: "scheduler", + additionalFlags: []string{ + "config", + }, + }, + { + command: "user", + additionalFlags: []string{ + "token", + "client-name", + }, + }, + } + + for _, test := range tests { + expectedFlags := append(commonFlags, test.additionalFlags...) + cmdtestutil.AssertSubCommandHasFlags(t, subCmds, test.command, expectedFlags...) + } +} + +func TestKubeConfigSubCommandsThatCreateFilesWithFlags(t *testing.T) { + + commonFlags := []string{ + "--apiserver-advertise-address=1.2.3.4", + "--apiserver-bind-port=1234", + } + + var tests = []struct { + command string + additionalFlags []string + expectedFiles []string + }{ + { + command: "all", + additionalFlags: []string{"--node-name=valid-nome-name"}, + expectedFiles: []string{ + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.KubeletKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + }, + }, + { + command: "admin", + expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName}, + }, + { + command: "kubelet", + additionalFlags: []string{"--node-name=valid-nome-name"}, + expectedFiles: []string{kubeadmconstants.KubeletKubeConfigFileName}, + }, + { + command: "controller-manager", + expectedFiles: []string{kubeadmconstants.ControllerManagerKubeConfigFileName}, + }, + { + command: "scheduler", + expectedFiles: []string{kubeadmconstants.SchedulerKubeConfigFileName}, + }, + } + + var kubeConfigAssertions = map[string]struct { + clientName string + organizations []string + }{ + kubeadmconstants.AdminKubeConfigFileName: { + clientName: "kubernetes-admin", + organizations: []string{kubeadmconstants.MastersGroup}, + }, + kubeadmconstants.KubeletKubeConfigFileName: { + clientName: "system:node:valid-nome-name", + organizations: []string{kubeadmconstants.NodesGroup}, + }, + kubeadmconstants.ControllerManagerKubeConfigFileName: { + clientName: kubeadmconstants.ControllerManagerUser, + }, + kubeadmconstants.SchedulerKubeConfigFileName: { + clientName: kubeadmconstants.SchedulerUser, + }, + } + + for _, test := range tests { + + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca certs to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Retrives ca cert for assertions + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + t.Fatalf("couldn't retrive ca cert: %v", err) + } + + // Get subcommands working in the temporary directory + subCmds := getKubeConfigSubCommands(nil, tmpdir) + + // Execute the subcommand + certDirFlag := fmt.Sprintf("--cert-dir=%s", pkidir) + allFlags := append(commonFlags, certDirFlag) + allFlags = append(allFlags, test.additionalFlags...) + cmdtestutil.RunSubCommand(t, subCmds, test.command, allFlags...) + + // Checks that requested files are there + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) + + // Checks contents of generated files + for _, file := range test.expectedFiles { + + // reads generated files + config, err := clientcmd.LoadFromFile(filepath.Join(tmpdir, file)) + if err != nil { + t.Errorf("Couldn't load generated kubeconfig file: %v", err) + } + + // checks that CLI flags are properly propagated and kubeconfig properties are correct + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + + expectedClientName := kubeConfigAssertions[file].clientName + expectedOrganizations := kubeConfigAssertions[file].organizations + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, expectedClientName, expectedOrganizations...) + + } + } +} + +func TestKubeConfigSubCommandsThatCreateFilesWithConfigFile(t *testing.T) { + + var tests = []struct { + command string + expectedFiles []string + }{ + { + command: "all", + expectedFiles: []string{ + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.KubeletKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + }, + }, + { + command: "admin", + expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName}, + }, + { + command: "kubelet", + expectedFiles: []string{kubeadmconstants.KubeletKubeConfigFileName}, + }, + { + command: "controller-manager", + expectedFiles: []string{kubeadmconstants.ControllerManagerKubeConfigFileName}, + }, + { + command: "scheduler", + expectedFiles: []string{kubeadmconstants.SchedulerKubeConfigFileName}, + }, + } + + var kubeConfigAssertions = map[string]struct { + clientName string + organizations []string + }{ + kubeadmconstants.AdminKubeConfigFileName: { + clientName: "kubernetes-admin", + organizations: []string{kubeadmconstants.MastersGroup}, + }, + kubeadmconstants.KubeletKubeConfigFileName: { + clientName: "system:node:valid-node-name", + organizations: []string{kubeadmconstants.NodesGroup}, + }, + kubeadmconstants.ControllerManagerKubeConfigFileName: { + clientName: kubeadmconstants.ControllerManagerUser, + }, + kubeadmconstants.SchedulerKubeConfigFileName: { + clientName: kubeadmconstants.SchedulerUser, + }, + } + + for _, test := range tests { + + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca certs to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Retrives ca cert for assertions + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + t.Fatalf("couldn't retrive ca cert: %v", err) + } + + // Adds a master configuration file + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + CertificatesDir: pkidir, + NodeName: "valid-node-name", + } + cfgPath := testutil.SetupMasterConfigurationFile(t, tmpdir, cfg) + + // Get subcommands working in the temporary directory + subCmds := getKubeConfigSubCommands(nil, tmpdir) + + // Execute the subcommand + configFlag := fmt.Sprintf("--config=%s", cfgPath) + cmdtestutil.RunSubCommand(t, subCmds, test.command, configFlag) + + // Checks that requested files are there + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) + + // Checks contents of generated files + for _, file := range test.expectedFiles { + + // reads generated files + config, err := clientcmd.LoadFromFile(filepath.Join(tmpdir, file)) + if err != nil { + t.Errorf("Couldn't load generated kubeconfig file: %v", err) + } + + // checks that config file properties are properly propagated and kubeconfig properties are correct + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + + expectedClientName := kubeConfigAssertions[file].clientName + expectedOrganizations := kubeConfigAssertions[file].organizations + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, expectedClientName, expectedOrganizations...) + + } + } +} + +func TestKubeConfigSubCommandsThatWritesToOut(t *testing.T) { + + // Temporary folders for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca cert to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Retrives ca cert for assertions + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + t.Fatalf("couldn't retrive ca cert: %v", err) + } + + commonFlags := []string{ + "--apiserver-advertise-address=1.2.3.4", + "--apiserver-bind-port=1234", + "--client-name=myUser", + fmt.Sprintf("--cert-dir=%s", pkidir), + } + + var tests = []struct { + command string + withClientCert bool + withToken bool + additionalFlags []string + }{ + { // Test user subCommand withClientCert + command: "user", + withClientCert: true, + }, + { // Test user subCommand withToken + withToken: true, + command: "user", + additionalFlags: []string{"--token=123456"}, + }, + } + + for _, test := range tests { + buf := new(bytes.Buffer) + + // Get subcommands working in the temporary directory + subCmds := getKubeConfigSubCommands(buf, tmpdir) + + // Execute the subcommand + allFlags := append(commonFlags, test.additionalFlags...) + cmdtestutil.RunSubCommand(t, subCmds, test.command, allFlags...) + + // reads kubeconfig written to stdout + config, err := clientcmd.Load(buf.Bytes()) + if err != nil { + t.Errorf("Couldn't read kubeconfig file from buffer: %v", err) + continue + } + + // checks that CLI flags are properly propagated + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + + if test.withClientCert { + // checks that kubeconfig files have expected client cert + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser") + } + + if test.withToken { + // checks that kubeconfig files have expected token + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "123456") + } + } +} diff --git a/cmd/kubeadm/app/phases/kubeconfig/BUILD b/cmd/kubeadm/app/phases/kubeconfig/BUILD index d5038b310ec..2164a903bb9 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/BUILD +++ b/cmd/kubeadm/app/phases/kubeconfig/BUILD @@ -5,6 +5,7 @@ licenses(["notice"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -15,6 +16,7 @@ go_library( ], tags = ["automanaged"], deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", @@ -36,3 +38,20 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["kubeconfig_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", + "//cmd/kubeadm/test:go_default_library", + "//cmd/kubeadm/test/certs:go_default_library", + "//cmd/kubeadm/test/kubeconfig:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index e3df1d43083..f73b8aa0372 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -20,79 +20,104 @@ import ( "bytes" "crypto/x509" "fmt" + "io" "os" "path/filepath" + "crypto/rsa" + "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" ) -// BuildConfigProperties holds some simple information about how this phase should build the KubeConfig object -type BuildConfigProperties struct { - CertDir string - ClientName string - Organization []string - APIServer string - Token string - MakeClientCerts bool +// clientCertAuth struct holds info required to build a client certificate to provide authentication info in a kubeconfig object +type clientCertAuth struct { + CaKey *rsa.PrivateKey + Organizations []string } -// TODO: Make an integration test for this function that runs after the certificates phase -// and makes sure that those two phases work well together... +// tokenAuth struct holds info required to use a token to provide authentication info in a kubeconfig object +type tokenAuth struct { + Token string +} -// TODO: Integration test cases: -// /etc/kubernetes/{admin,kubelet}.conf don't exist => generate kubeconfig files -// /etc/kubernetes/{admin,kubelet}.conf and certs in /etc/kubernetes/pki exist => don't touch anything as long as everything's valid -// /etc/kubernetes/{admin,kubelet}.conf exist but the server URL is invalid in those files => error -// /etc/kubernetes/{admin,kubelet}.conf exist but the CA cert doesn't match what's in the pki dir => error -// /etc/kubernetes/{admin,kubelet}.conf exist but not certs => certs will be generated and conflict with the kubeconfig files => error +// kubeConfigSpec struct holds info required to build a KubeConfig object +type kubeConfigSpec struct { + CaCert *x509.Certificate + APIServer string + ClientName string + TokenAuth *tokenAuth + ClientCertAuth *clientCertAuth +} -// CreateInitKubeConfigFiles is called from the main init and does the work for the default phase behaviour -func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir, nodeName string) error { +// CreateInitKubeConfigFiles will create and write to disk all kubeconfig files necessary in the kubeadm init phase +// to establish the control plane, including also the admin kubeconfig file. +// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned. +func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles( + outDir, + cfg, + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.KubeletKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + ) +} - // Create a lightweight specification for what the files should look like - filesToCreateFromSpec := map[string]BuildConfigProperties{ - kubeadmconstants.AdminKubeConfigFileName: { - ClientName: "kubernetes-admin", - APIServer: masterEndpoint, - CertDir: pkiDir, - Organization: []string{kubeadmconstants.MastersGroup}, - MakeClientCerts: true, - }, - kubeadmconstants.KubeletKubeConfigFileName: { - ClientName: fmt.Sprintf("system:node:%s", nodeName), - APIServer: masterEndpoint, - CertDir: pkiDir, - Organization: []string{kubeadmconstants.NodesGroup}, - MakeClientCerts: true, - }, - kubeadmconstants.ControllerManagerKubeConfigFileName: { - ClientName: kubeadmconstants.ControllerManagerUser, - APIServer: masterEndpoint, - CertDir: pkiDir, - MakeClientCerts: true, - }, - kubeadmconstants.SchedulerKubeConfigFileName: { - ClientName: kubeadmconstants.SchedulerUser, - APIServer: masterEndpoint, - CertDir: pkiDir, - MakeClientCerts: true, - }, +// CreateAdminKubeConfigFile create a kubeconfig file for the admin to use and for kubeadm itself. +// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. +func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles(outDir, cfg, kubeadmconstants.AdminKubeConfigFileName) +} + +// CreateKubeletKubeConfigFile create a kubeconfig file for the Kubelet to use. +// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. +func CreateKubeletKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles(outDir, cfg, kubeadmconstants.KubeletKubeConfigFileName) +} + +// CreateControllerManagerKubeConfigFile create a kubeconfig file for the ControllerManager to use. +// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. +func CreateControllerManagerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles(outDir, cfg, kubeadmconstants.ControllerManagerKubeConfigFileName) +} + +// CreateSchedulerKubeConfigFile create a create a kubeconfig file for the Scheduler to use. +// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. +func CreateSchedulerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles(outDir, cfg, kubeadmconstants.SchedulerKubeConfigFileName) +} + +// createKubeConfigFiles creates all the requested kubeconfig files. +// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned. +func createKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration, kubeConfigFileNames ...string) error { + + // gets the KubeConfigSpecs, actualized for the current MasterConfiguration + specs, err := getKubeConfigSpecs(cfg) + if err != nil { + return err } - // Loop through all specs for kubeconfig files and create them if necessary - for filename, config := range filesToCreateFromSpec { - kubeconfig, err := buildKubeConfig(config) + for _, kubeConfigFileName := range kubeConfigFileNames { + // retrives the KubeConfigSpec for given kubeConfigFileName + spec, exists := specs[kubeConfigFileName] + if !exists { + return fmt.Errorf("couldn't retrive KubeConfigSpec for %s", kubeConfigFileName) + } + + // builds the KubeConfig object + config, err := buildKubeConfigFromSpec(spec) if err != nil { return err } - kubeConfigFilePath := filepath.Join(outDir, filename) - err = writeKubeconfigToDiskIfNotExists(kubeConfigFilePath, kubeconfig) + // writes the KubeConfig to disk if it not exists + err = createKubeConfigFileIfNotExists(outDir, kubeConfigFileName, config) if err != nil { return err } @@ -101,87 +126,128 @@ func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir, nodeName string) return nil } -// GetKubeConfigBytesFromSpec takes properties how to build a KubeConfig file and then returns the bytes of that file -func GetKubeConfigBytesFromSpec(config BuildConfigProperties) ([]byte, error) { - kubeconfig, err := buildKubeConfig(config) - if err != nil { - return []byte{}, err - } +// getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current MasterConfiguration +// NB. this methods holds the information about how kubeadm creates kubeconfig files. +func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeConfigSpec, error) { - kubeConfigBytes, err := clientcmd.Write(*kubeconfig) - if err != nil { - return []byte{}, err - } - return kubeConfigBytes, nil -} - -// buildKubeConfig creates a kubeconfig object from some commonly specified properties in the struct above -func buildKubeConfig(config BuildConfigProperties) (*clientcmdapi.Config, error) { - - // Try to load ca.crt and ca.key from the PKI directory - caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(config.CertDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) } - // If this file should have client certs, generate one from the spec - if config.MakeClientCerts { - certConfig := certutil.Config{ - CommonName: config.ClientName, - Organization: config.Organization, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - } - cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, certConfig) - if err != nil { - return nil, fmt.Errorf("failure while creating %s client certificate [%v]", certConfig.CommonName, err) - } - return kubeconfigutil.CreateWithCerts( - config.APIServer, + var kubeConfigSpec = map[string]*kubeConfigSpec{ + kubeadmconstants.AdminKubeConfigFileName: { + CaCert: caCert, + APIServer: cfg.GetMasterEndpoint(), + ClientName: "kubernetes-admin", + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + Organizations: []string{kubeadmconstants.MastersGroup}, + }, + }, + kubeadmconstants.KubeletKubeConfigFileName: { + CaCert: caCert, + APIServer: cfg.GetMasterEndpoint(), + ClientName: fmt.Sprintf("system:node:%s", cfg.NodeName), + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + Organizations: []string{kubeadmconstants.NodesGroup}, + }, + }, + kubeadmconstants.ControllerManagerKubeConfigFileName: { + CaCert: caCert, + APIServer: cfg.GetMasterEndpoint(), + ClientName: kubeadmconstants.ControllerManagerUser, + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + }, + }, + kubeadmconstants.SchedulerKubeConfigFileName: { + CaCert: caCert, + APIServer: cfg.GetMasterEndpoint(), + ClientName: kubeadmconstants.SchedulerUser, + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + }, + }, + } + + return kubeConfigSpec, nil +} + +// buildKubeConfigFromSpec creates a kubeconfig object for the given kubeConfigSpec +func buildKubeConfigFromSpec(spec *kubeConfigSpec) (*clientcmdapi.Config, error) { + + // If this kubeconfing should use token + if spec.TokenAuth != nil { + // create a kubeconfig with a token + return kubeconfigutil.CreateWithToken( + spec.APIServer, "kubernetes", - config.ClientName, - certutil.EncodeCertPEM(caCert), - certutil.EncodePrivateKeyPEM(key), - certutil.EncodeCertPEM(cert), + spec.ClientName, + certutil.EncodeCertPEM(spec.CaCert), + spec.TokenAuth.Token, ), nil } - // otherwise, create a kubeconfig with a token - return kubeconfigutil.CreateWithToken( - config.APIServer, + // otherwise, create a client certs + clientCertConfig := certutil.Config{ + CommonName: spec.ClientName, + Organization: spec.ClientCertAuth.Organizations, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CaCert, spec.ClientCertAuth.CaKey, clientCertConfig) + if err != nil { + return nil, fmt.Errorf("failure while creating %s client certificate: %v", spec.ClientName, err) + } + + // create a kubeconfig with the client certs + return kubeconfigutil.CreateWithCerts( + spec.APIServer, "kubernetes", - config.ClientName, - certutil.EncodeCertPEM(caCert), - config.Token, + spec.ClientName, + certutil.EncodeCertPEM(spec.CaCert), + certutil.EncodePrivateKeyPEM(clientKey), + certutil.EncodeCertPEM(clientCert), ), nil } -// writeKubeconfigToDiskIfNotExists saves the KubeConfig struct to disk if there isn't any file at the given path +// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path. // If there already is a KubeConfig file at the given path; kubeadm tries to load it and check if the values in the // existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date, // but if a file exists but has old content or isn't a kubeconfig file, this function returns an error. -func writeKubeconfigToDiskIfNotExists(filename string, expectedConfig *clientcmdapi.Config) error { +func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error { + kubeConfigFilePath := filepath.Join(outDir, filename) + // Check if the file exist, and if it doesn't, just write it to disk - if _, err := os.Stat(filename); os.IsNotExist(err) { - return kubeconfigutil.WriteToDisk(filename, expectedConfig) + if _, err := os.Stat(kubeConfigFilePath); os.IsNotExist(err) { + err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config) + if err != nil { + return fmt.Errorf("failed to save kubeconfig file %s on disk: %v", kubeConfigFilePath, err) + } + + fmt.Printf("[kubeconfig] Wrote KubeConfig file to disk: %q\n", filename) + return nil } // The kubeconfig already exists, let's check if it has got the same CA and server URL - currentConfig, err := clientcmd.LoadFromFile(filename) + currentConfig, err := clientcmd.LoadFromFile(kubeConfigFilePath) if err != nil { - return fmt.Errorf("failed to load kubeconfig that already exists on disk [%v]", err) + return fmt.Errorf("failed to load kubeconfig file %s that already exists on disk: %v", kubeConfigFilePath, err) } - expectedCtx := expectedConfig.CurrentContext - expectedCluster := expectedConfig.Contexts[expectedCtx].Cluster + expectedCtx := config.CurrentContext + expectedCluster := config.Contexts[expectedCtx].Cluster currentCtx := currentConfig.CurrentContext currentCluster := currentConfig.Contexts[currentCtx].Cluster + // If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale - if !bytes.Equal(currentConfig.Clusters[currentCluster].CertificateAuthorityData, expectedConfig.Clusters[expectedCluster].CertificateAuthorityData) { - return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", filename) + if !bytes.Equal(currentConfig.Clusters[currentCluster].CertificateAuthorityData, config.Clusters[expectedCluster].CertificateAuthorityData) { + return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath) } // If the current API Server location on disk doesn't match the expected API server, error out because we have a file, but it's stale - if currentConfig.Clusters[currentCluster].Server != expectedConfig.Clusters[expectedCluster].Server { - return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong API Server URL", filename) + if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server { + return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong API Server URL", kubeConfigFilePath) } // kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid) @@ -191,3 +257,64 @@ func writeKubeconfigToDiskIfNotExists(filename string, expectedConfig *clientcmd return nil } + +// WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info - to the given writer. +func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName string) error { + + // creates the KubeConfigSpecs, actualized for the current MasterConfiguration + caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) + } + + spec := &kubeConfigSpec{ + ClientName: clientName, + APIServer: cfg.GetMasterEndpoint(), + CaCert: caCert, + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + }, + } + + return writeKubeConfigFromSpec(out, spec) +} + +// WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer. +func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName, token string) error { + + // creates the KubeConfigSpecs, actualized for the current MasterConfiguration + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) + } + + spec := &kubeConfigSpec{ + ClientName: clientName, + APIServer: cfg.GetMasterEndpoint(), + CaCert: caCert, + TokenAuth: &tokenAuth{ + Token: token, + }, + } + + return writeKubeConfigFromSpec(out, spec) +} + +// writeKubeConfigFromSpec creates a kubeconfig object from a kubeConfigSpec and writes it to the given writer. +func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec) error { + + // builds the KubeConfig object + config, err := buildKubeConfigFromSpec(spec) + if err != nil { + return err + } + + // writes the KubeConfig to disk if it not exists + configBytes, err := clientcmd.Write(*config) + if err != nil { + return fmt.Errorf("failure while serializing admin kubeconfig: %v", err) + } + + fmt.Fprintln(out, string(configBytes)) + return nil +} diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go new file mode 100644 index 00000000000..610c6e4929e --- /dev/null +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -0,0 +1,435 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeconfig + +import ( + "bytes" + "crypto/rsa" + "crypto/x509" + "fmt" + "io" + "os" + "reflect" + "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" + certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" + kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig" +) + +func TestGetKubeConfigSpecsFailsIfCADoesntExists(t *testing.T) { + // Create temp folder for the test case (without a CA) + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Master Configuration pointing to the pkidir folder + cfg := &kubeadmapi.MasterConfiguration{ + CertificatesDir: tmpdir, + } + + // Executes getKubeConfigSpecs + if _, err := getKubeConfigSpecs(cfg); err == nil { + t.Error("getKubeConfigSpecs didnt failed when expected") + } +} + +func TestGetKubeConfigSpecs(t *testing.T) { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca certs to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Creates a Master Configuration pointing to the pkidir folder + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + CertificatesDir: pkidir, + NodeName: "valid-node-name", + } + + // Executes getKubeConfigSpecs + specs, err := getKubeConfigSpecs(cfg) + if err != nil { + t.Fatal("getKubeConfigSpecs failed!") + } + + var assertions = []struct { + kubeConfigFile string + clientName string + organizations []string + }{ + { + kubeConfigFile: kubeadmconstants.AdminKubeConfigFileName, + clientName: "kubernetes-admin", + organizations: []string{kubeadmconstants.MastersGroup}, + }, + { + kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName, + clientName: fmt.Sprintf("system:node:%s", cfg.NodeName), + organizations: []string{kubeadmconstants.NodesGroup}, + }, + { + kubeConfigFile: kubeadmconstants.ControllerManagerKubeConfigFileName, + clientName: kubeadmconstants.ControllerManagerUser, + }, + { + kubeConfigFile: kubeadmconstants.SchedulerKubeConfigFileName, + clientName: kubeadmconstants.SchedulerUser, + }, + } + + for _, assertion := range assertions { + + // assert the spec for the kubeConfigFile exists + if spec, ok := specs[assertion.kubeConfigFile]; ok { + + // Assert clientName + if spec.ClientName != assertion.clientName { + t.Errorf("getKubeConfigSpecs for %s clientName is %s, expected %s", assertion.kubeConfigFile, spec.ClientName, assertion.clientName) + } + + // Assert Organizations + if spec.ClientCertAuth == nil || !reflect.DeepEqual(spec.ClientCertAuth.Organizations, assertion.organizations) { + t.Errorf("getKubeConfigSpecs for %s Organizations is %v, expected %v", assertion.kubeConfigFile, spec.ClientCertAuth.Organizations, assertion.organizations) + } + + // Asserts MasterConfiguration values injected into spec + if spec.APIServer != cfg.GetMasterEndpoint() { + t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer address into spec for %s", assertion.kubeConfigFile) + } + + // Asserts CA certs and CA keys loaded into specs + if spec.CaCert == nil { + t.Errorf("getKubeConfigSpecs didn't loaded CaCert into spec for %s!", assertion.kubeConfigFile) + } + if spec.ClientCertAuth == nil || spec.ClientCertAuth.CaKey == nil { + t.Errorf("getKubeConfigSpecs didn't loaded CaKey into spec for %s!", assertion.kubeConfigFile) + } + } else { + t.Errorf("getKubeConfigSpecs didn't create spec for %s ", assertion.kubeConfigFile) + } + } +} + +func TestBuildKubeConfigFromSpecWithClientAuth(t *testing.T) { + // Creates a CA + caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) + + // Executes buildKubeConfigFromSpec passing a KubeConfigSpec wiht a ClientAuth + config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "myClientName", "myOrg1", "myOrg2") + + // Asserts spec data are propagated to the kubeconfig + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myClientName", "myOrg1", "myOrg2") +} + +func TestBuildKubeConfigFromSpecWithTokenAuth(t *testing.T) { + // Creates a CA + caCert, _ := certstestutil.SetupCertificateAuthorithy(t) + + // Executes buildKubeConfigFromSpec passing a KubeConfigSpec wiht a Token + config := setupdKubeConfigWithTokenAuth(t, caCert, "https://1.2.3.4:1234", "myClientName", "123456") + + // Asserts spec data are propagated to the kubeconfig + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myClientName", "123456") +} + +func TestCreateKubeConfigFileIfNotExists(t *testing.T) { + + // Creates a CAs + caCert, caKey := certstestutil.SetupCertificateAuthorithy(t) + anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthorithy(t) + + // build kubeconfigs (to be used to test kubeconfigs equality/not equality) + config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "myOrg1", "myOrg2") + configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "myOrg1", "myOrg2") + configWithAnotherClusterAddress := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://3.4.5.6:3456", "myOrg1", "myOrg2") + + var tests = []struct { + existingKubeConfig *clientcmdapi.Config + kubeConfig *clientcmdapi.Config + expectedError bool + }{ + { // if there is no existing KubeConfig, creates the kubeconfig + kubeConfig: config, + }, + { // if KubeConfig is equal to the existingKubeConfig - refers to the same cluster -, use the existing (Test idempotency) + existingKubeConfig: config, + kubeConfig: config, + }, + { // if KubeConfig is not equal to the existingKubeConfig - refers to the another cluster (a cluster with another Ca) -, raise error + existingKubeConfig: config, + kubeConfig: configWithAnotherClusterCa, + expectedError: true, + }, + { // if KubeConfig is not equal to the existingKubeConfig - refers to the another cluster (a cluster with another address) -, raise error + existingKubeConfig: config, + kubeConfig: configWithAnotherClusterAddress, + expectedError: true, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Writes the existing kubeconfig file to disk + if test.existingKubeConfig != nil { + if err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.existingKubeConfig); err != nil { + t.Errorf("createKubeConfigFileIfNotExists failed") + } + } + + // Writes the KubeConfig file to disk + err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.kubeConfig) + if test.expectedError && err == nil { + t.Errorf("createKubeConfigFileIfNotExists didn't failed when expected to fail") + } + if !test.expectedError && err != nil { + t.Errorf("createKubeConfigFileIfNotExists failed") + } + + // Assert creted files is there + testutil.AssertFileExists(t, tmpdir, "test.conf") + } +} + +func TestCreateKubeconfigFilesAndWrappers(t *testing.T) { + var tests = []struct { + createKubeConfigFunction func(outDir string, cfg *kubeadmapi.MasterConfiguration) error + expectedFiles []string + expectedError bool + }{ + { // Test createKubeConfigFiles fails for unknown kubeconfig is requested + createKubeConfigFunction: func(outDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createKubeConfigFiles(outDir, cfg, "unknown.conf") + }, + expectedError: true, + }, + { // Test CreateInitKubeConfigFiles (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateInitKubeConfigFiles, + expectedFiles: []string{ + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.KubeletKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + }, + }, + { // Test CreateAdminKubeConfigFile (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateAdminKubeConfigFile, + expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName}, + }, + { // Test CreateKubeletKubeConfigFile (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateKubeletKubeConfigFile, + expectedFiles: []string{kubeadmconstants.KubeletKubeConfigFileName}, + }, + { // Test CreateControllerManagerKubeConfigFile (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateControllerManagerKubeConfigFile, + expectedFiles: []string{kubeadmconstants.ControllerManagerKubeConfigFileName}, + }, + { // Test createKubeConfigFile (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateSchedulerKubeConfigFile, + expectedFiles: []string{kubeadmconstants.SchedulerKubeConfigFileName}, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca certs to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Creates a Master Configuration pointing to the pkidir folder + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + CertificatesDir: pkidir, + } + + // Execs the createKubeConfigFunction + err := test.createKubeConfigFunction(tmpdir, cfg) + if test.expectedError && err == nil { + t.Errorf("createKubeConfigFunction didn't failed when expected to fail") + continue + } + if !test.expectedError && err != nil { + t.Errorf("createKubeConfigFunction failed") + continue + } + + // Assert expected files are there + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) + } +} + +func TestWriteKubeConfigFailsIfCADoesntExists(t *testing.T) { + + // Temporary folders for the test case (without a CA) + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Master Configuration pointing to the tmpdir folder + cfg := &kubeadmapi.MasterConfiguration{ + CertificatesDir: tmpdir, + } + + var tests = []struct { + writeKubeConfigFunction func(out io.Writer) error + }{ + { // Test WriteKubeConfigWithClientCert + writeKubeConfigFunction: func(out io.Writer) error { + return WriteKubeConfigWithClientCert(out, cfg, "myUser") + }, + }, + { // Test WriteKubeConfigWithToken + writeKubeConfigFunction: func(out io.Writer) error { + return WriteKubeConfigWithToken(out, cfg, "myUser", "12345") + }, + }, + } + + for _, test := range tests { + buf := new(bytes.Buffer) + + // executes writeKubeConfigFunction + if err := test.writeKubeConfigFunction(buf); err == nil { + t.Error("writeKubeConfigFunction didnt failed when expected") + } + } +} + +func TestWriteKubeConfig(t *testing.T) { + + // Temporary folders for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Adds a pki folder with a ca cert to the temp folder + pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir) + + // Retrives ca cert for assertions + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + t.Fatalf("couldn't retrive ca cert: %v", err) + } + + // Creates a Master Configuration pointing to the pkidir folder + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + CertificatesDir: pkidir, + } + + var tests = []struct { + writeKubeConfigFunction func(out io.Writer) error + withClientCert bool + withToken bool + }{ + { // Test WriteKubeConfigWithClientCert + writeKubeConfigFunction: func(out io.Writer) error { + return WriteKubeConfigWithClientCert(out, cfg, "myUser") + }, + withClientCert: true, + }, + { // Test WriteKubeConfigWithToken + writeKubeConfigFunction: func(out io.Writer) error { + return WriteKubeConfigWithToken(out, cfg, "myUser", "12345") + }, + withToken: true, + }, + } + + for _, test := range tests { + buf := new(bytes.Buffer) + + // executes writeKubeConfigFunction + if err := test.writeKubeConfigFunction(buf); err != nil { + t.Error("writeKubeConfigFunction failed") + continue + } + + // reads kubeconfig written to stdout + config, err := clientcmd.Load(buf.Bytes()) + if err != nil { + t.Errorf("Couldn't read kubeconfig file from buffer: %v", err) + continue + } + + // checks that CLI flags are properly propagated + kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) + + if test.withClientCert { + // checks that kubeconfig files have expected client cert + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser") + } + + if test.withToken { + // checks that kubeconfig files have expected token + kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "12345") + } + } +} + +// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth +func setupdKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey, APIServer, clientName string, organizations ...string) *clientcmdapi.Config { + spec := &kubeConfigSpec{ + CaCert: caCert, + APIServer: APIServer, + ClientName: clientName, + ClientCertAuth: &clientCertAuth{ + CaKey: caKey, + Organizations: organizations, + }, + } + + config, err := buildKubeConfigFromSpec(spec) + if err != nil { + t.Fatal("buildKubeConfigFromSpec failed!") + } + + return config +} + +// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token +func setupdKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, APIServer, clientName, token string) *clientcmdapi.Config { + spec := &kubeConfigSpec{ + CaCert: caCert, + APIServer: APIServer, + ClientName: clientName, + TokenAuth: &tokenAuth{ + Token: token, + }, + } + + config, err := buildKubeConfigFromSpec(spec) + if err != nil { + t.Fatal("buildKubeConfigFromSpec failed!") + } + + return config +} diff --git a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go index 8f380ede1a3..acbba287e16 100644 --- a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go @@ -96,7 +96,6 @@ func WriteToDisk(filename string, kubeconfig *clientcmdapi.Config) error { return err } - fmt.Printf("[kubeconfig] Wrote KubeConfig file to disk: %q\n", filename) return nil } diff --git a/cmd/kubeadm/test/BUILD b/cmd/kubeadm/test/BUILD new file mode 100644 index 00000000000..27b845a4ca3 --- /dev/null +++ b/cmd/kubeadm/test/BUILD @@ -0,0 +1,39 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["util.go"], + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", + "//cmd/kubeadm/test/certs:go_default_library", + "//vendor/github.com/renstrom/dedent:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//cmd/kubeadm/test/certs:all-srcs", + "//cmd/kubeadm/test/cmd:all-srcs", + "//cmd/kubeadm/test/kubeconfig:all-srcs", + ], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/test/certs/BUILD b/cmd/kubeadm/test/certs/BUILD new file mode 100644 index 00000000000..a12d0b2fe19 --- /dev/null +++ b/cmd/kubeadm/test/certs/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["util.go"], + tags = ["automanaged"], + deps = ["//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/test/certs/util.go b/cmd/kubeadm/test/certs/util.go new file mode 100644 index 00000000000..30ef1d303ae --- /dev/null +++ b/cmd/kubeadm/test/certs/util.go @@ -0,0 +1,79 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs + +import ( + "crypto/rsa" + "crypto/x509" + "testing" + + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" +) + +// SetupCertificateAuthorithy is a utility function for kubeadm testing that creates a +// CertificateAuthorithy cert/key pair +func SetupCertificateAuthorithy(t *testing.T) (*x509.Certificate, *rsa.PrivateKey) { + caCert, caKey, err := pkiutil.NewCertificateAuthority() + if err != nil { + t.Fatalf("failure while generating CA certificate and key: %v", err) + } + + return caCert, caKey +} + +// AssertCertificateIsSignedByCa is a utility function for kubeadm testing that asserts if a given certificate is signed +// by the expected CA +func AssertCertificateIsSignedByCa(t *testing.T, cert *x509.Certificate, signingCa *x509.Certificate) { + if err := cert.CheckSignatureFrom(signingCa); err != nil { + t.Error("cert is not signed by signing CA as expected") + } +} + +// AssertCertificateHasCommonName is a utility function for kubeadm testing that asserts if a given certificate has +// the expected SubjectCommonName +func AssertCertificateHasCommonName(t *testing.T, cert *x509.Certificate, commonName string) { + if cert.Subject.CommonName != commonName { + t.Errorf("cert has Subject.CommonName %s, expected %s", cert.Subject.CommonName, commonName) + } +} + +// AssertCertificateHasOrganizations is a utility function for kubeadm testing that asserts if a given certificate has +// the expected Subject.Organization +func AssertCertificateHasOrganizations(t *testing.T, cert *x509.Certificate, organizations ...string) { + for _, organization := range organizations { + found := false + for i := range cert.Subject.Organization { + if cert.Subject.Organization[i] == organization { + found = true + } + } + if !found { + t.Errorf("cert does not contain Subject.Organization %s as expected", organization) + } + } +} + +// AssertCertificateHasClientAuthUsage is a utility function for kubeadm testing that asserts if a given certificate has +// the expected ExtKeyUsageClientAuth +func AssertCertificateHasClientAuthUsage(t *testing.T, cert *x509.Certificate) { + for i := range cert.ExtKeyUsage { + if cert.ExtKeyUsage[i] == x509.ExtKeyUsageClientAuth { + return + } + } + t.Error("cert has not ClientAuth usage as expected") +} diff --git a/cmd/kubeadm/test/cmd/BUILD b/cmd/kubeadm/test/cmd/BUILD index 19265aa5538..3a489fa9f50 100644 --- a/cmd/kubeadm/test/cmd/BUILD +++ b/cmd/kubeadm/test/cmd/BUILD @@ -12,6 +12,7 @@ go_library( name = "go_default_library", srcs = ["util.go"], tags = ["automanaged"], + deps = ["//vendor/github.com/spf13/cobra:go_default_library"], ) go_test( diff --git a/cmd/kubeadm/test/cmd/util.go b/cmd/kubeadm/test/cmd/util.go index 1dab5e33a7f..f11b1a27187 100644 --- a/cmd/kubeadm/test/cmd/util.go +++ b/cmd/kubeadm/test/cmd/util.go @@ -20,6 +20,9 @@ import ( "bytes" "fmt" "os/exec" + "testing" + + "github.com/spf13/cobra" ) // Forked from test/e2e/framework because the e2e framework is quite bloated @@ -37,3 +40,34 @@ func RunCmd(command string, args ...string) (string, string, error) { } return stdout, stderr, nil } + +// RunSubCommand is a utility function for kubeadm testing that executes a Cobra sub command +func RunSubCommand(t *testing.T, subCmds []*cobra.Command, command string, args ...string) { + subCmd := getSubCommand(t, subCmds, command) + subCmd.SetArgs(args) + if err := subCmd.Execute(); err != nil { + t.Fatalf("Could not execute subcommand: %s", command) + } +} + +// AssertSubCommandHasFlags is a utility function for kubeadm testing that assert if a Cobra sub command has expected flags +func AssertSubCommandHasFlags(t *testing.T, subCmds []*cobra.Command, command string, flags ...string) { + subCmd := getSubCommand(t, subCmds, command) + + for _, flag := range flags { + if subCmd.Flags().Lookup(flag) == nil { + t.Errorf("Could not find expecte flag %s for command %s", flag, command) + } + } +} + +func getSubCommand(t *testing.T, subCmds []*cobra.Command, name string) *cobra.Command { + for _, subCmd := range subCmds { + if subCmd.Name() == name { + return subCmd + } + } + t.Fatalf("Unable to find sub command %s", name) + + return nil +} diff --git a/cmd/kubeadm/test/kubeconfig/BUILD b/cmd/kubeadm/test/kubeconfig/BUILD new file mode 100644 index 00000000000..972410efdbb --- /dev/null +++ b/cmd/kubeadm/test/kubeconfig/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["util.go"], + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/test/certs:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/test/kubeconfig/util.go b/cmd/kubeadm/test/kubeconfig/util.go new file mode 100644 index 00000000000..daad6892282 --- /dev/null +++ b/cmd/kubeadm/test/kubeconfig/util.go @@ -0,0 +1,100 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeconfig + +import ( + "crypto/x509" + "encoding/pem" + "testing" + + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" +) + +// AssertKubeConfigCurrentCluster is a utility function for kubeadm testing that asserts if the CurrentCluster in +// the given KubeConfig object contains refers to a specific cluster +func AssertKubeConfigCurrentCluster(t *testing.T, config *clientcmdapi.Config, expectedAPIServerAddress string, expectedAPIServerCaCert *x509.Certificate) { + currentContext := config.Contexts[config.CurrentContext] + currentCluster := config.Clusters[currentContext.Cluster] + + // Assert expectedAPIServerAddress + if currentCluster.Server != expectedAPIServerAddress { + t.Errorf("kubeconfig.currentCluster.Server is [%s], expected [%s]", currentCluster.Server, expectedAPIServerAddress) + } + + // Assert the APIServerCaCert + if len(currentCluster.CertificateAuthorityData) == 0 { + t.Error("kubeconfig.currentCluster.CertificateAuthorityData is empty, expected not empty") + return + } + + block, _ := pem.Decode(currentCluster.CertificateAuthorityData) + currentAPIServerCaCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Errorf("kubeconfig.currentCluster.CertificateAuthorityData is not a valid CA: %v", err) + return + } + + if !currentAPIServerCaCert.Equal(expectedAPIServerCaCert) { + t.Errorf("kubeconfig.currentCluster.CertificateAuthorityData not correspond to the expected CA cert") + } +} + +// AssertKubeConfigCurrentAuthInfoWithClientCert is a utility function for kubeadm testing that asserts if the CurrentAuthInfo in +// the given KubeConfig object contains a clientCert that refers to a specific client name, is signed by the expected CA, includes the expected organizations +func AssertKubeConfigCurrentAuthInfoWithClientCert(t *testing.T, config *clientcmdapi.Config, signinCa *x509.Certificate, expectedClientName string, expectedOrganizations ...string) { + currentContext := config.Contexts[config.CurrentContext] + currentAuthInfo := config.AuthInfos[currentContext.AuthInfo] + + // assert clientCert + if len(currentAuthInfo.ClientCertificateData) == 0 { + t.Error("kubeconfig.currentAuthInfo.ClientCertificateData is empty, expected not empty") + return + } + + block, _ := pem.Decode(config.AuthInfos[currentContext.AuthInfo].ClientCertificateData) + currentClientCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Errorf("kubeconfig.currentAuthInfo.ClientCertificateData is not a valid CA: %v", err) + return + } + + // Asserts the clientCert is signed by the signinCa + certstestutil.AssertCertificateIsSignedByCa(t, currentClientCert, signinCa) + + // Asserts the clientCert has ClientAuth ExtKeyUsage + certstestutil.AssertCertificateHasClientAuthUsage(t, currentClientCert) + + // Asserts the clientCert has expected expectedUserName as CommonName + certstestutil.AssertCertificateHasCommonName(t, currentClientCert, expectedClientName) + + // Asserts the clientCert has expected Organizations + certstestutil.AssertCertificateHasOrganizations(t, currentClientCert, expectedOrganizations...) +} + +// AssertKubeConfigCurrentAuthInfoWithToken is a utility function for kubeadm testing that asserts if the CurrentAuthInfo in +// the given KubeConfig object refers to expected token +func AssertKubeConfigCurrentAuthInfoWithToken(t *testing.T, config *clientcmdapi.Config, expectedClientName, expectedToken string) { + currentContext := config.Contexts[config.CurrentContext] + currentAuthInfo := config.AuthInfos[currentContext.AuthInfo] + + // assert token + if currentAuthInfo.Token != expectedToken { + t.Errorf("kubeconfig.currentAuthInfo.Token [%s], expected [%s]", currentAuthInfo.Token, expectedToken) + return + } +} diff --git a/cmd/kubeadm/test/util.go b/cmd/kubeadm/test/util.go new file mode 100644 index 00000000000..fe8ea38f781 --- /dev/null +++ b/cmd/kubeadm/test/util.go @@ -0,0 +1,126 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "html/template" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/renstrom/dedent" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + certtestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" +) + +// SetupTempDir is a utility function for kubeadm testing, that creates a temporary directory +// NB. it is up to the caller to cleanup the folder at the end of the test with defer os.RemoveAll(tmpdir) +func SetupTempDir(t *testing.T) string { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + + return tmpdir +} + +// SetupMasterConfigurationFile is a utility function for kubeadm testing that writes a master configuration file +// into /config subfolder of a given temporary directory. +// The funtion returns the path of the created master configuration file. +func SetupMasterConfigurationFile(t *testing.T, tmpdir string, cfg *kubeadmapi.MasterConfiguration) string { + + cfgPath := filepath.Join(tmpdir, "config/masterconfig.yaml") + if err := os.MkdirAll(filepath.Dir(cfgPath), os.FileMode(0755)); err != nil { + t.Fatalf("Couldn't create cfgDir") + } + + cfgTemplate := template.Must(template.New("init").Parse(dedent.Dedent(` + apiVersion: kubeadm.k8s.io/v1alpha1 + kind: MasterConfiguration + certificatesDir: {{.CertificatesDir}} + api: + advertiseAddress: {{.API.AdvertiseAddress}} + bindPort: {{.API.BindPort}} + nodeName: {{.NodeName}} + `))) + + f, err := os.Create(cfgPath) + if err != nil { + t.Fatalf("error creating masterconfig file %s: %v", cfgPath, err) + } + + err = cfgTemplate.Execute(f, cfg) + if err != nil { + t.Fatalf("error generating masterconfig file %s: %v", cfgPath, err) + } + f.Close() + + return cfgPath +} + +// SetupPkiDirWithCertificateAuthorithy is a utility function for kubeadm testing that creates a +// CertificateAuthorithy cert/key pair into /pki subfolder of a given temporary directory. +// The funtion returns the path of the created pki. +func SetupPkiDirWithCertificateAuthorithy(t *testing.T, tmpdir string) string { + caCert, caKey := certtestutil.SetupCertificateAuthorithy(t) + + certDir := filepath.Join(tmpdir, "pki") + if err := pkiutil.WriteCertAndKey(certDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil { + t.Fatalf("failure while saving CA certificate and key: %v", err) + } + + return certDir +} + +// AssertFilesCount is a utility function for kubeadm testing that asserts if the given folder contains +// count files. +func AssertFilesCount(t *testing.T, dirName string, count int) { + files, err := ioutil.ReadDir(dirName) + if err != nil { + t.Fatalf("Couldn't read files from tmpdir: %s", err) + } + + countFiles := 0 + for _, f := range files { + if !f.IsDir() { + countFiles++ + } + } + + if countFiles != count { + t.Errorf("dir does contains %d, %d expected", len(files), count) + for _, f := range files { + t.Error(f.Name()) + } + } +} + +// AssertFileExists is a utility function for kubeadm testing that asserts if the given folder contains +// the given files. +func AssertFileExists(t *testing.T, dirName string, fileNames ...string) { + for _, fileName := range fileNames { + path := filepath.Join(dirName, fileName) + + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Errorf("file %s does not exist", fileName) + } + } +}