From f223d814da6274303aa2731b5d8f32b72e71c0ec Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 18 Aug 2016 13:38:18 +0100 Subject: [PATCH 01/32] Initial version of kubeadm --- cmd/kubeadm/app/kubeadm.go | 65 +++++++++ cmd/kubeadm/{dummy.go => kubeadm.go} | 10 +- hack/.linted_packages | 1 + pkg/kubeadm/.gitignore | 1 + pkg/kubeadm/api/types.go | 44 ++++++ pkg/kubeadm/cmd/cmd.go | 97 +++++++++++++ pkg/kubeadm/cmd/init.go | 108 +++++++++++++++ pkg/kubeadm/cmd/join.go | 87 ++++++++++++ pkg/kubeadm/cmd/manual.go | 195 ++++++++++++++++++++++++++ pkg/kubeadm/cmd/user.go | 34 +++++ pkg/kubeadm/master/addons.go | 93 +++++++++++++ pkg/kubeadm/master/apiclient.go | 111 +++++++++++++++ pkg/kubeadm/master/discovery.go | 111 +++++++++++++++ pkg/kubeadm/master/kubeconfig.go | 57 ++++++++ pkg/kubeadm/master/manifests.go | 196 +++++++++++++++++++++++++++ pkg/kubeadm/master/pki.go | 180 ++++++++++++++++++++++++ pkg/kubeadm/master/tokens.go | 59 ++++++++ pkg/kubeadm/node/csr.go | 90 ++++++++++++ pkg/kubeadm/node/discovery.go | 87 ++++++++++++ pkg/kubeadm/util/kubeconfig.go | 103 ++++++++++++++ pkg/kubeadm/util/tokens.go | 91 +++++++++++++ 21 files changed, 1818 insertions(+), 2 deletions(-) create mode 100644 cmd/kubeadm/app/kubeadm.go rename cmd/kubeadm/{dummy.go => kubeadm.go} (79%) create mode 100644 pkg/kubeadm/.gitignore create mode 100644 pkg/kubeadm/api/types.go create mode 100644 pkg/kubeadm/cmd/cmd.go create mode 100644 pkg/kubeadm/cmd/init.go create mode 100644 pkg/kubeadm/cmd/join.go create mode 100644 pkg/kubeadm/cmd/manual.go create mode 100644 pkg/kubeadm/cmd/user.go create mode 100644 pkg/kubeadm/master/addons.go create mode 100644 pkg/kubeadm/master/apiclient.go create mode 100644 pkg/kubeadm/master/discovery.go create mode 100644 pkg/kubeadm/master/kubeconfig.go create mode 100644 pkg/kubeadm/master/manifests.go create mode 100644 pkg/kubeadm/master/pki.go create mode 100644 pkg/kubeadm/master/tokens.go create mode 100644 pkg/kubeadm/node/csr.go create mode 100644 pkg/kubeadm/node/discovery.go create mode 100644 pkg/kubeadm/util/kubeconfig.go create mode 100644 pkg/kubeadm/util/tokens.go diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go new file mode 100644 index 00000000000..584909353c4 --- /dev/null +++ b/cmd/kubeadm/app/kubeadm.go @@ -0,0 +1,65 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/pflag" + + "k8s.io/kubernetes/pkg/kubeadm/cmd" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/logs" +) + +var CommandLine *pflag.FlagSet + +// TODO(phase2) use componentconfig +// we need some params for testing etc, let's keep these hidden for now +func getEnvParams() map[string]string { + globalPrefix := os.Getenv("KUBE_PREFIX_ALL") + if globalPrefix == "" { + globalPrefix = "/etc/kubernetes" + } + + envParams := map[string]string{ + "prefix": globalPrefix, + "host_pki_path": path.Join(globalPrefix, "pki"), + "hyperkube_image": "gcr.io/google_containers/hyperkube:v1.4.0-alpha.3", + "discovery_image": "dgoodwin/kubediscovery:latest", + } + + for k, _ := range envParams { + if v := os.Getenv(fmt.Sprintf("KUBE_%s", strings.ToUpper(k))); v != "" { + envParams[k] = v + } + } + + return envParams +} + +func Run() error { + CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + logs.InitLogs() + defer logs.FlushLogs() + + cmd := cmd.NewKubeadmCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr, getEnvParams()) + return cmd.Execute() +} diff --git a/cmd/kubeadm/dummy.go b/cmd/kubeadm/kubeadm.go similarity index 79% rename from cmd/kubeadm/dummy.go rename to cmd/kubeadm/kubeadm.go index 17711df5045..6aca5a18f3d 100644 --- a/cmd/kubeadm/dummy.go +++ b/cmd/kubeadm/kubeadm.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +17,14 @@ limitations under the License. package main import ( - _ "github.com/square/go-jose" + "os" + + "k8s.io/kubernetes/cmd/kubeadm/app" ) func main() { + if err := app.Run(); err != nil { + os.Exit(1) + } + os.Exit(0) } diff --git a/hack/.linted_packages b/hack/.linted_packages index da961ba18c5..3fd57c57f7e 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -243,3 +243,4 @@ test/integration/openshift test/soak/cauldron test/soak/serve_hostnames third_party/forked/golang/expansion +cmd/kubeadm diff --git a/pkg/kubeadm/.gitignore b/pkg/kubeadm/.gitignore new file mode 100644 index 00000000000..a8a20d9f1ba --- /dev/null +++ b/pkg/kubeadm/.gitignore @@ -0,0 +1 @@ +kubeadm diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go new file mode 100644 index 00000000000..e659162cb0e --- /dev/null +++ b/pkg/kubeadm/api/types.go @@ -0,0 +1,44 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadmapi + +type BootstrapParams struct { + // A struct with methods that implement Discover() + // kubeadm will do the CSR dance + Discovery *OutOfBandDiscovery + EnvParams map[string]string +} + +type OutOfBandDiscovery struct { + // 'join-node' side + ApiServerURLs string // comma separated + CaCertFile string + GivenToken string // dot-separated `.` set by the user + TokenID string // optional on master side, will be generated if not specified + Token []byte // optional on master side, will be generated if not specified + BearerToken string // set based on Token + // 'init-master' side + ApiServerDNSName string // optional, used in master bootstrap + ListenIP string // optional IP for master to listen on, rather than autodetect +} + +type ClusterInfo struct { + // TODO Kind, apiVersion + // TODO clusterId, fetchedTime, expiredTime + CertificateAuthorities []string `json:"certificateAuthorities"` + Endpoints []string `json:"endpoints"` +} diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go new file mode 100644 index 00000000000..2fe3ae0f169 --- /dev/null +++ b/pkg/kubeadm/cmd/cmd.go @@ -0,0 +1,97 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/flag" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, envParams map[string]string) *cobra.Command { + cmds := &cobra.Command{ + Use: "kubeadm", + Short: "kubeadm: bootstrap a secure kubernetes cluster easily.", + Long: dedent.Dedent(` + kubeadm: bootstrap a secure kubernetes cluster easily. + + ┌──────────────────────────────────────────────────────────┐ + │ KUBEADM IS ALPHA, DO NOT USE IT FOR PRODUCTION CLUSTERS! │ + │ │ + │ But, please try it out! Give us feedback at: │ + │ https://github.com/kubernetes/kubernetes/issues │ + │ and at-mention @kubernetes/sig-cluster-lifecycle │ + └──────────────────────────────────────────────────────────┘ + + Example usage: + + Create a two-machine cluster with one master (which controls the cluster), + and one node (where workloads, like pods and replica sets run). + + ┌──────────────────────────────────────────────────────────┐ + │ On the first machine │ + ├──────────────────────────────────────────────────────────┤ + │ master# kubeadm init master │ + │ Your token is: │ + └──────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────┐ + │ On the second machine │ + ├──────────────────────────────────────────────────────────┤ + │ node# kubeadm join node --token= │ + └──────────────────────────────────────────────────────────┘ + + You can then repeat the second step on as many other machines as you like. + + `), + } + // TODO figure out how to avoid running as root + // + // TODO also print the alpha warning when running any commands, as well as + // in the help text. + // + // TODO detect interactive vs non-interactive use and adjust output accordingly + // i.e. make it automation friendly + // + // TODO create an bastraction that defines files and the content that needs to + // be written to disc and write it all in one go at the end as we have a lot of + // crapy little files written from different parts of this code; this could also + // be useful for testing + + bootstrapParams := &kubeadmapi.BootstrapParams{ + Discovery: &kubeadmapi.OutOfBandDiscovery{ + // TODO this type no longer makes sense here + }, + EnvParams: envParams, + } + + cmds.ResetFlags() + cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) + + cmds.AddCommand(NewCmdInit(out, bootstrapParams)) + cmds.AddCommand(NewCmdJoin(out, bootstrapParams)) + cmds.AddCommand(NewCmdUser(out, bootstrapParams)) + cmds.AddCommand(NewCmdManual(out, bootstrapParams)) + + return cmds +} diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go new file mode 100644 index 00000000000..5c7a541900b --- /dev/null +++ b/pkg/kubeadm/cmd/init.go @@ -0,0 +1,108 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + init_done_msgf = dedent.Dedent(` + Kubernetes master initialised successfully! + + You can connect any number of nodes by running: + + kubeadm join --token %s %s + `) +) + +func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "init --token [--listen-ip ]", + Short: "Run this on the first server you deploy onto.", + Run: func(cmd *cobra.Command, args []string) { + err := RunInit(out, cmd, args, params) + cmdutil.CheckErr(err) + }, + } + + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + `(optional) IP address to listen on, in case autodetection fails.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + + return cmd +} + +func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + if params.Discovery.ListenIP == "" { + ip, err := kubeadmutil.GetDefaultHostIP() + if err != nil { + return err + } + params.Discovery.ListenIP = ip + } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { + return err + } + if err := kubemaster.WriteStaticPodManifests(params); err != nil { + return err + } + caKey, caCert, err := kubemaster.CreatePKIAssets(params) + if err != nil { + return err + } + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + if err != nil { + return err + } + for name, kubeconfig := range kubeconfigs { + if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + return err + } + } + + client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"]) + if err != nil { + return err + } + + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil { + return err + } + + if err := kubemaster.CreateEssentialAddons(params, client); err != nil { + return err + } + + // TODO use templates to reference struct fields directly as order of args is fragile + fmt.Fprintf(out, init_done_msgf, + params.Discovery.GivenToken, + params.Discovery.ListenIP, + ) + + return nil +} diff --git a/pkg/kubeadm/cmd/join.go b/pkg/kubeadm/cmd/join.go new file mode 100644 index 00000000000..5c8f7a1ffb3 --- /dev/null +++ b/pkg/kubeadm/cmd/join.go @@ -0,0 +1,87 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubenode "k8s.io/kubernetes/pkg/kubeadm/node" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + join_done_msgf = dedent.Dedent(` + Node join complete: + * Certificate signing request sent to master and response + received. + * Kubelet informed of new secure connection details. + + Run 'kubectl get nodes' on the master to see this node join. + `) +) + +func NewCmdJoin(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "join", + Short: "Run this on other servers to join an existing cluster.", + Run: func(cmd *cobra.Command, args []string) { + err := RunJoin(out, cmd, args, params) + cmdutil.CheckErr(err) + }, + } + + // TODO this should become `kubeadm join --token=<...> ` + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + `Comma separated list of API server URLs. Typically this might be just + https://:8080/`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + + return cmd +} + +func RunJoin(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(params) + if !ok { + if err != nil { + return fmt.Errorf("%s (see --help)\n", err) + } + return fmt.Errorf("Must specify --token (see --help)\n") + } + if params.Discovery.ApiServerURLs == "" { + return fmt.Errorf("Must specify --api-server-urls (see --help)\n") + } + + kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(params) + if err != nil { + return err + } + + err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + if err != nil { + return err + } + + fmt.Fprintf(out, join_done_msgf) + return nil +} diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go new file mode 100644 index 00000000000..05902986f66 --- /dev/null +++ b/pkg/kubeadm/cmd/manual.go @@ -0,0 +1,195 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" + kubenode "k8s.io/kubernetes/pkg/kubeadm/node" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + netutil "k8s.io/kubernetes/pkg/util/net" + // TODO: cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + manual_done_msgf = dedent.Dedent(` + Master initialization complete: + + * Static pods written and kubelet's kubeconfig written. + * Kubelet should start soon. Try 'systemctl restart kubelet' + or equivalent if it doesn't. + + CA cert is written to: + /etc/kubernetes/pki/ca.pem. + + **Please copy this file (scp, rsync or through other means) to + all your nodes and then run on them**: + + kubeadm manual bootstrap join-node --ca-cert-file \ + --token %s --api-server-urls https://%s:443/ + `) +) + +// TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken +// may be we should make it the same and ask user to pass dot-separated tokens +// in any of the modes; we could also enable discovery API in the manual mode just +// as well, there is no reason we shouldn't let user mix and match modes, unless +// it is too difficult to support + +func NewCmdManual(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "manual", + Short: "Advanced, less-automated functionality, for power users.", + // TODO put example usage in the Long description here + } + cmd.AddCommand(NewCmdManualBootstrap(out, params)) + return cmd +} + +func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "bootstrap", + Short: "Manually bootstrap a cluster 'out-of-band'", + Long: dedent.Dedent(` + Manually bootstrap a cluster 'out-of-band', by generating and distributing a CA + certificate to all your servers and specifying and (list of) API server URLs. + `), + Run: func(cmd *cobra.Command, args []string) { + }, + } + cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, params)) + cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, params)) + + return cmd +} + +func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "init-master", + Short: "Manually bootstrap a master 'out-of-band'", + Long: dedent.Dedent(` + Manually bootstrap a master 'out-of-band'. + Will create TLS certificates and set up static pods for Kubernetes master + components. + `), + RunE: func(cmd *cobra.Command, args []string) error { + if params.Discovery.ListenIP == "" { + ip, err := netutil.ChooseHostInterface() + if err != nil { + return fmt.Errorf("Unable to autodetect IP address [%s], please specify with --listen-ip", err) + } + params.Discovery.ListenIP = ip + } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { + return err + } + if err := kubemaster.WriteStaticPodManifests(params); err != nil { + return err + } + caKey, caCert, err := kubemaster.CreatePKIAssets(params) + if err != nil { + return err + } + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + if err != nil { + return err + } + for name, kubeconfig := range kubeconfigs { + if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + out.Write([]byte(fmt.Sprintf("Unable to write admin for master:\n%s\n", err))) + return nil + } + } + + // TODO use templates to reference struct fields directly as order of args is fragile + fmt.Fprintf(out, manual_done_msgf, + params.Discovery.BearerToken, + params.Discovery.ListenIP, + ) + + return nil + }, + } + + params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", "", + `(optional) DNS name for the API server, will be encoded into + subjectAltName in the resulting (generated) TLS certificates`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + `(optional) IP address to listen on, in case autodetection fails.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + + return cmd +} + +func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "join-node", + Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", + + Run: func(cmd *cobra.Command, args []string) { + if params.Discovery.CaCertFile == "" { + out.Write([]byte(fmt.Sprintf("Must specify --ca-cert-file (see --help)\n"))) + return + } + + if params.Discovery.ApiServerURLs == "" { + out.Write([]byte(fmt.Sprintf("Must specify --api-server-urls (see --help)\n"))) + return + } + + kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) + if err != nil { + out.Write([]byte(fmt.Sprintf("Failed to perform TLS bootstrap: %s\n", err))) + return + } + //fmt.Println("recieved signed certificate from the API server, will write `/etc/kubernetes/kubelet.conf`...") + + err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + if err != nil { + out.Write([]byte(fmt.Sprintf("Unable to write config for node:\n%s\n", err))) + return + } + out.Write([]byte(dedent.Dedent(` + Node join complete: + * Certificate signing request sent to master and response + received. + * Kubelet informed of new secure connection details. + + Run 'kubectl get nodes' on the master to see this node join. + + `))) + }, + } + cmd.PersistentFlags().StringVarP(¶ms.Discovery.CaCertFile, "ca-cert-file", "", "", + `Path to a CA cert file in PEM format. The same CA cert must be distributed to + all servers.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + `Comma separated list of API server URLs. Typically this might be just + https://:8080/`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + + return cmd +} diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go new file mode 100644 index 00000000000..3d30f50d6e5 --- /dev/null +++ b/pkg/kubeadm/cmd/user.go @@ -0,0 +1,34 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "io" + + "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func NewCmdUser(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "user", + Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap + Run: func(cmd *cobra.Command, args []string) { + }, + } + return cmd +} diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go new file mode 100644 index 00000000000..97911ede831 --- /dev/null +++ b/pkg/kubeadm/master/addons.go @@ -0,0 +1,93 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "fmt" + "path" + + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { + privilegedTrue := true + return api.PodSpec{ + SecurityContext: &api.PodSecurityContext{HostNetwork: true}, + Containers: []api.Container{{ + Name: "kube-proxy", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "proxy", + "--kubeconfig=/run/kubeconfig", + COMPONENT_LOGLEVEL, + }, + SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, + VolumeMounts: []api.VolumeMount{ + { + Name: "dbus", + MountPath: "/var/run/dbus", + ReadOnly: false, + }, + { + // TODO there are handful of clever options to get around this, but it's + // easier to just mount kubelet's config here; we should probably just + // make sure that proxy reads the token and CA cert from /run/secrets + // and accepts `--master` at the same time + // + // clever options include: + // - do CSR dance and create kubeconfig and mount it as secrete + // - create a service account with a second secret enconding kubeconfig + // - use init container to convert known information to kubeconfig + // - ...whatever + Name: "kubeconfig", + MountPath: "/run/kubeconfig", + ReadOnly: false, + }, + }, + }}, + Volumes: []api.Volume{ + { + Name: "kubeconfig", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: path.Join(params.EnvParams["prefix"], "kubelet.conf")}, + }, + }, + { + Name: "dbus", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: "/var/run/dbus"}, + }, + }, + }, + } +} + +func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { + kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) + + if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { + return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + } + + fmt.Println(" created essential addon: kube-proxy") + + // TODO should we wait for it to become ready at least on the master? + + return nil +} diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go new file mode 100644 index 00000000000..4a2d404d9bf --- /dev/null +++ b/pkg/kubeadm/master/apiclient.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "fmt" + "time" + + "k8s.io/kubernetes/pkg/api" + unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/extensions" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + "k8s.io/kubernetes/pkg/util/wait" +) + +func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) { + adminClientConfig, err := clientcmd.NewDefaultClientConfig( + *adminConfig, + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + if err != nil { + return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + } + + fmt.Println(" created API client configuration") + + client, err := clientset.NewForConfig(adminClientConfig) + if err != nil { + return nil, fmt.Errorf(" failed to create API client [%s]", err) + } + + fmt.Println(" created API client, waiting for the control plane to become ready") + + start := time.Now() + wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + cs, err := client.ComponentStatuses().List(api.ListOptions{}) + if err != nil { + return false, nil + } + if len(cs.Items) < 3 { + fmt.Println(" not all control plane components are ready yet") + return false, nil + } + for _, item := range cs.Items { + for _, condition := range item.Conditions { + if condition.Type != api.ComponentHealthy { + fmt.Printf(" control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions) + return false, nil + } + } + } + + fmt.Printf(" all control plane components are healthy after %s seconds\n", time.Since(start).Seconds()) + return true, nil + }) + + // TODO may be also check node status + return client, nil +} + +func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet { + l := map[string]string{"component": daemonName, "tier": "node"} + return &extensions.DaemonSet{ + ObjectMeta: api.ObjectMeta{Name: daemonName}, + Spec: extensions.DaemonSetSpec{ + Selector: &unversionedapi.LabelSelector{MatchLabels: l}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{Labels: l}, + Spec: podSpec, + }, + }, + } +} + +func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) *extensions.Deployment { + l := map[string]string{"name": deploymentName} + return &extensions.Deployment{ + ObjectMeta: api.ObjectMeta{Name: deploymentName}, + Spec: extensions.DeploymentSpec{ + Replicas: replicas, + Selector: &unversionedapi.LabelSelector{MatchLabels: l}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{Labels: l}, + Spec: podSpec, + }, + }, + } +} + +func TaintMaster(*clientset.Clientset) error { + // TODO + annotations := make(map[string]string) + annotations[api.TaintsAnnotationKey] = "" + return nil +} diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go new file mode 100644 index 00000000000..76caac90090 --- /dev/null +++ b/pkg/kubeadm/master/discovery.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "crypto/x509" + "encoding/hex" + "encoding/json" + "fmt" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +type kubeDiscovery struct { + Deployment *extensions.Deployment + Secret *api.Secret +} + +const ( + kubeDiscoverynName = "kube-discovery" + kubeDiscoverySecretName = "clusterinfo" +) + +func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) map[string][]byte { + // TODO ListenIP is probably not the right now, although it's best we have right now + // if user provides a DNS name, or anything else, we should use that, may be it's really + // the list of all SANs (minus internal DNS names and service IP)? + + var ( + data = map[string][]byte{} + endpointList = []string{} + tokenMap = map[string]string{} + ) + + endpointList = append(endpointList, fmt.Sprintf("https://%s:443", params.Discovery.ListenIP)) + tokenMap[params.Discovery.TokenID] = hex.EncodeToString(params.Discovery.Token) + + data["endpoint-list.json"], _ = json.Marshal(endpointList) + data["token-map.json"], _ = json.Marshal(tokenMap) + data["ca.pem"] = certutil.EncodeCertPEM(caCert) + + return data +} + +func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { + return api.PodSpec{ + SecurityContext: &api.PodSecurityContext{HostNetwork: true}, // TODO we should just use map it to a host port + Containers: []api.Container{{ + Name: kubeDiscoverynName, + Image: params.EnvParams["discovery_image"], + Command: []string{"/usr/bin/kube-discovery"}, + VolumeMounts: []api.VolumeMount{{ + Name: kubeDiscoverySecretName, + MountPath: "/tmp/secret", // TODO use a shared constant + ReadOnly: true, + }}, + }}, + Volumes: []api.Volume{{ + Name: kubeDiscoverySecretName, + VolumeSource: api.VolumeSource{ + Secret: &api.SecretVolumeSource{SecretName: kubeDiscoverySecretName}, + }}, + }, + } +} + +func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery { + // TODO pin to master + return kubeDiscovery{ + Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)), + Secret: &api.Secret{ + ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, + Type: api.SecretTypeOpaque, + Data: encodeKubeDiscoverySecretData(params, caCert), + }, + } +} + +func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error { + kd := newKubeDiscovery(params, caCert) + + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { + return fmt.Errorf(" failed to create %q deployment", kubeDiscoverynName) + } + if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil { + return fmt.Errorf(" failed to create %q secret", kubeDiscoverySecretName) + } + + fmt.Println(" created essential addon: kube-discovery") + + // TODO we should probably wait for the pod to become ready + + return nil +} diff --git a/pkg/kubeadm/master/kubeconfig.go b/pkg/kubeadm/master/kubeconfig.go new file mode 100644 index 00000000000..9cd9506310b --- /dev/null +++ b/pkg/kubeadm/master/kubeconfig.go @@ -0,0 +1,57 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + + // TODO: "k8s.io/client-go/client/tools/clientcmd/api" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +func CreateCertsAndConfigForClients(params *kubeadmapi.BootstrapParams, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { + + basicClientConfig := kubeadmutil.CreateBasicClientConfig( + "kubernetes", + fmt.Sprintf("https://%s:443", params.Discovery.ListenIP), + certutil.EncodeCertPEM(caCert), + ) + + configs := map[string]*clientcmdapi.Config{} + + for _, client := range clientNames { + key, cert, err := newClientKeyAndCert(caCert, caKey) + if err != nil { + return nil, fmt.Errorf(" failure while creating %s client certificate - %s", client, err) + } + config := kubeadmutil.MakeClientConfigWithCerts( + basicClientConfig, + "kubernetes", + client, + certutil.EncodePrivateKeyPEM(key), + certutil.EncodeCertPEM(cert), + ) + configs[client] = config + } + + return configs, nil +} diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go new file mode 100644 index 00000000000..b8810be2b6f --- /dev/null +++ b/pkg/kubeadm/master/manifests.go @@ -0,0 +1,196 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" + api "k8s.io/kubernetes/pkg/api/v1" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/intstr" +) + +// Static pod definitions in golang form are included below so that `kubeadm +// init master` and `kubeadm manual bootstrap master` can get going. + +const ( + COMPONENT_LOGLEVEL = "--v=4" + SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=10.16.0.0/12" + CLUSTER_NAME = "--cluster-name=kubernetes" + MASTER = "--master=127.0.0.1:8080" +) + +// TODO look into what this really means, scheduler prints it for some reason +// +//E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' + +func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { + staticPodSpecs := map[string]api.Pod{ + // TODO this needs a volume + "etcd": componentPod(api.Container{ + Command: []string{ + "/usr/local/bin/etcd", + "--listen-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", + "--advertise-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", + "--data-dir=/var/etcd/data", + }, + Image: "gcr.io/google_containers/etcd:2.2.1", // TODO parametrise + LivenessProbe: componentProbe(2379, "/health"), + Name: "etcd-server", + Resources: componentResources("200m"), + }), + // TODO bind-mount certs in + "kube-apiserver": componentPod(api.Container{ + Name: "kube-apiserver", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "apiserver", + "--address=127.0.0.1", + "--etcd-servers=http://127.0.0.1:2379", + "--cloud-provider=fake", // TODO parametrise + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", + SERVICE_CLUSTER_IP_RANGE, + "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--client-ca-file=/etc/kubernetes/pki/ca.pem", + "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", + "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--secure-port=443", + "--allow-privileged", + COMPONENT_LOGLEVEL, + "--token-auth-file=/etc/kubernetes/pki/tokens.csv", + }, + VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + LivenessProbe: componentProbe(8080, "/healthz"), + Resources: componentResources("250m"), + }, pkiVolume(params)), + "kube-controller-manager": componentPod(api.Container{ + Name: "kube-controller-manager", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "controller-manager", + "--leader-elect", + MASTER, + CLUSTER_NAME, + "--root-ca-file=/etc/kubernetes/pki/ca.pem", + "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", + "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", + COMPONENT_LOGLEVEL, + }, + VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + LivenessProbe: componentProbe(10252, "/healthz"), + Resources: componentResources("200m"), + }, pkiVolume(params)), + "kube-scheduler": componentPod(api.Container{ + Name: "kube-scheduler", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "scheduler", + "--leader-elect", + MASTER, + COMPONENT_LOGLEVEL, + }, + LivenessProbe: componentProbe(10251, "/healthz"), + Resources: componentResources("100m"), + }), + } + + manifestsPath := path.Join(params.EnvParams["prefix"], "manifests") + if err := os.MkdirAll(manifestsPath, 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) + } + for name, spec := range staticPodSpecs { + filename := path.Join(manifestsPath, name+".json") + serialized, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return fmt.Errorf(" failed to marshall manifest for %q to JSON [%s]", name, err) + } + if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), filename); err != nil { + return fmt.Errorf(" failed to create static pod manifest file for %q (%q) [%s]", name, filename, err) + } + } + return nil +} + +func pkiVolume(params *kubeadmapi.BootstrapParams) api.Volume { + return api.Volume{ + Name: "pki", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: params.EnvParams["host_pki_path"]}, + }, + } +} + +func pkiVolumeMount() api.VolumeMount { + return api.VolumeMount{ + Name: "pki", + MountPath: "/etc/kubernetes/pki", + ReadOnly: true, + } +} + +func componentResources(cpu string) api.ResourceRequirements { + return api.ResourceRequirements{ + Requests: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse(cpu), + }, + } +} + +func componentProbe(port int, path string) *api.Probe { + return &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Host: "127.0.0.1", + Path: path, + Port: intstr.FromInt(port), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 15, + } +} + +func componentPod(container api.Container, volumes ...api.Volume) api.Pod { + return api.Pod{ + TypeMeta: unversioned.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: api.ObjectMeta{ + Name: container.Name, + Namespace: "kube-system", + Labels: map[string]string{"component": container.Name, "tier": "control-plane"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{container}, + HostNetwork: true, + Volumes: volumes, + }, + } +} diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go new file mode 100644 index 00000000000..6c23c90a5de --- /dev/null +++ b/pkg/kubeadm/master/pki.go @@ -0,0 +1,180 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + "path" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +/* +func errorf(f string, err error, vargs ...string) error { + return fmt.Errorf(" %s [%s]", fmt.Sprintf(f, v...), err) +} +*/ + +func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + } + + config := certutil.CertConfig{ + CommonName: "kubernetes", + } + + cert, err := certutil.NewSelfSignedCACert(config, key) + if err != nil { + return nil, nil, fmt.Errorf("unable to create self-singed certificate [%s]", err) + } + + return key, cert, nil +} + +func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) + } + // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly + altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1")) + altNames.DNSNames = append(altNames.DNSNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + ) + + config := certutil.CertConfig{ + CommonName: "kube-apiserver", + AltNames: altNames, + } + cert, err := certutil.NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to sing certificate [%s]", err) + } + + return key, cert, nil +} + +func newClientKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + } + + config := certutil.CertConfig{ + CommonName: "kubernetes-admin", + } + cert, err := certutil.NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to sign certificate [%s]", err) + } + + return key, cert, nil +} + +func writeKeysAndCert(pkiPath string, name string, key *rsa.PrivateKey, cert *x509.Certificate) error { + var ( + publicKeyPath = path.Join(pkiPath, fmt.Sprintf("%s-pub.pem", name)) + privateKeyPath = path.Join(pkiPath, fmt.Sprintf("%s-key.pem", name)) + certificatePath = path.Join(pkiPath, fmt.Sprintf("%s.pem", name)) + ) + + if key != nil { + if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil { + return fmt.Errorf("unable to write private key file (%q) [%s]", privateKeyPath, err) + } + if pubKey, err := certutil.EncodePublicKeyPEM(&key.PublicKey); err == nil { + if err := certutil.WriteKey(publicKeyPath, pubKey); err != nil { + return fmt.Errorf("unable to write public key file (%q) [%s]", publicKeyPath, err) + } + } else { + return fmt.Errorf("unable to encode public key to PEM [%s]", err) + } + } + + if cert != nil { + if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil { + return fmt.Errorf("unable to write certificate file (%q) [%s]", err) + } + } + + return nil +} + +func newServiceAccountKey() (*rsa.PrivateKey, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, err + } + return key, nil +} + +func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509.Certificate, error) { + var ( + err error + altNames certutil.AltNames // TODO actual SANs + ) + + if params.Discovery.ListenIP != "" { + altNames.IPs = append(altNames.IPs, net.ParseIP(params.Discovery.ListenIP)) + } + + if params.Discovery.ApiServerDNSName != "" { + altNames.DNSNames = append(altNames.DNSNames, params.Discovery.ApiServerDNSName) + } + + pkiPath := path.Join(params.EnvParams["host_pki_path"]) + + caKey, caCert, err := newCertificateAuthority() + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating CA keys and certificate - %s", err) + } + + if err := writeKeysAndCert(pkiPath, "ca", caKey, caCert); err != nil { + return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %s", err) + } + + apiKey, apiCert, err := newServerKeyAndCert(caCert, caKey, altNames) + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %s", err) + } + + if err := writeKeysAndCert(pkiPath, "apiserver", apiKey, apiCert); err != nil { + return nil, nil, fmt.Errorf(" failure while saving API server keys and certificate - %s", err) + } + + saKey, err := newServiceAccountKey() + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating service account signing keys [%s]", err) + } + + if err := writeKeysAndCert(pkiPath, "sa", saKey, nil); err != nil { + return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) + } + + // TODO print a summary of SANs used and checksums (signatures) of each of the certiicates + fmt.Println(" created keys and certificates in %q", params.EnvParams["host_pki_path"]) + return caKey, caCert, nil +} diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go new file mode 100644 index 00000000000..81d899807b0 --- /dev/null +++ b/pkg/kubeadm/master/tokens.go @@ -0,0 +1,59 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "bytes" + "fmt" + "os" + "path" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/uuid" +) + +func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(params) + if !ok { + if err != nil { + return err + } + err = kubeadmutil.GenerateToken(params) + if err != nil { + return err + } + fmt.Printf(" generated token: %q\n", params.Discovery.GivenToken) + } + + return nil +} + +func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { + tokenAuthFilePath := path.Join(params.EnvParams["host_pki_path"], "tokens.csv") + if err := generateTokenIfNeeded(params); err != nil { + return fmt.Errorf(" failed to generate token(s) [%s]", err) + } + if err := os.MkdirAll(path.Join(params.EnvParams["host_pki_path"]), 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) + } + serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) + if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil { + return fmt.Errorf(" failed to save token auth file (%q) [%s]", tokenAuthFilePath, err) + } + return nil +} diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go new file mode 100644 index 00000000000..e1da517d59d --- /dev/null +++ b/pkg/kubeadm/node/csr.go @@ -0,0 +1,90 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubenode + +import ( + "fmt" + "io/ioutil" + "strings" + + unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + "k8s.io/kubernetes/pkg/kubelet/util/csr" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +func getNodeName() string { + return "TODO" +} + +func PerformTLSBootstrapFromParams(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { + caCert, err := ioutil.ReadFile(params.Discovery.CaCertFile) + if err != nil { + return nil, fmt.Errorf(" failed to load CA certificate [%s]", err) + } + + return PerformTLSBootstrap(params, strings.Split(params.Discovery.ApiServerURLs, ",")[0], caCert) +} + +// Create a restful client for doing the certificate signing request. +func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { + // TODO try all the api servers until we find one that works + bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) + + nodeName := getNodeName() + + bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( + *kubeadmutil.MakeClientConfigWithToken( + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), params.Discovery.BearerToken, + ), + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + if err != nil { + return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + } + + client, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) + if err != nil { + return nil, fmt.Errorf(" failed to create API client [%s]", err) + } + csrClient := client.CertificateSigningRequests() + + fmt.Println(" created API client to obtain unique certificate for this node, generating keys and certificate signing request") + + key, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + return nil, fmt.Errorf(" failed to generating private key [%s]", err) + } + + cert, err := csr.RequestNodeCertificate(csrClient, key, nodeName) + if err != nil { + return nil, fmt.Errorf(" failed to request signed certificate from the API server [%s]", err) + } + + // TODO print some basic info about the cert + fmt.Println(" received signed certificate from the API server, generating kubelet configuration") + + finalConfig := kubeadmutil.MakeClientConfigWithCerts( + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), + key, cert, + ) + + return finalConfig, nil +} diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go new file mode 100644 index 00000000000..9609905b65e --- /dev/null +++ b/pkg/kubeadm/node/discovery.go @@ -0,0 +1,87 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubenode + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/square/go-jose" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { + firstURL := strings.Split(params.Discovery.ApiServerURLs, ",")[0] // TODO obviously we should do something better.. . + apiServerURL, err := url.Parse(firstURL) + if err != nil { + return nil, fmt.Errorf(" failed to parse given API server URL (%q) [%s]", firstURL, err) + } + + host, port := strings.Split(apiServerURL.Host, ":")[0], 9898 // TODO this is too naive + requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, params.Discovery.TokenID) + req, err := http.NewRequest("GET", requestURL, nil) + if err != nil { + return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) + } + + fmt.Println(" created cluster info discovery client, requesting info from %q", requestURL) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf(" failed to request cluster info [%s]", err) + } + buf := new(bytes.Buffer) + io.Copy(buf, res.Body) + res.Body.Close() + + object, err := jose.ParseSigned(buf.String()) + if err != nil { + return nil, fmt.Errorf(" failed to parse response as JWS object [%s]", err) + } + + fmt.Println(" cluster info object recieved, verifying signature using given token") + + output, err := object.Verify(params.Discovery.Token) + if err != nil { + return nil, fmt.Errorf(" failed to verify JWS signature of recieved cluster info object [%s]", err) + } + + clusterInfo := kubeadmapi.ClusterInfo{} + + if err := json.Unmarshal(output, &clusterInfo); err != nil { + return nil, fmt.Errorf(" failed to unmarshal recieved cluster info object [%s]", err) + } + + if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 { + return nil, fmt.Errorf(" cluster info object is invalid - no endpoint(s) and/or root CA certificate(s) found") + } + + // TODO print checksum of the CA certificate + fmt.Printf(" cluser info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) + + // TODO we need to configure the client to validate the server + // if it is signed by any of the returned certificates + apiServer := clusterInfo.Endpoints[0] + caCert := []byte(clusterInfo.CertificateAuthorities[0]) + + return PerformTLSBootstrap(params, apiServer, caCert) +} diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go new file mode 100644 index 00000000000..9c595d57580 --- /dev/null +++ b/pkg/kubeadm/util/kubeconfig.go @@ -0,0 +1,103 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadmutil + +import ( + "fmt" + "os" + "path" + + // TODO: "k8s.io/client-go/client/tools/clientcmd/api" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func CreateBasicClientConfig(clusterName string, serverURL string, caCert []byte) *clientcmdapi.Config { + cluster := clientcmdapi.NewCluster() + cluster.Server = serverURL + cluster.CertificateAuthorityData = caCert + + config := clientcmdapi.NewConfig() + config.Clusters[clusterName] = cluster + + return config +} + +func MakeClientConfigWithCerts(config *clientcmdapi.Config, clusterName string, userName string, clientKey []byte, clientCert []byte) *clientcmdapi.Config { + newConfig := config + name := fmt.Sprintf("%s@%s", userName, clusterName) + + authInfo := clientcmdapi.NewAuthInfo() + authInfo.ClientKeyData = clientKey + authInfo.ClientCertificateData = clientCert + + context := clientcmdapi.NewContext() + context.Cluster = clusterName + context.AuthInfo = userName + + newConfig.AuthInfos[userName] = authInfo + newConfig.Contexts[name] = context + newConfig.CurrentContext = name + + return newConfig +} + +func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, userName string, token string) *clientcmdapi.Config { + newConfig := config + name := fmt.Sprintf("%s@%s", userName, clusterName) + + authInfo := clientcmdapi.NewAuthInfo() + authInfo.Token = token + + context := clientcmdapi.NewContext() + context.Cluster = clusterName + context.AuthInfo = userName + + newConfig.AuthInfos[userName] = authInfo + newConfig.Contexts[name] = context + newConfig.CurrentContext = name + + return newConfig +} + +// kubeadm is responsible for writing the following kubeconfig file, which +// kubelet should be waiting for. Help user avoid foot-shooting by refusing to +// write a file that has already been written (the kubelet will be up and +// running in that case - they'd need to stop the kubelet, remove the file, and +// start it again in that case). + +func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { + filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + // Create and open the file, only if it does not already exist. + f, err := os.OpenFile( + filename, + os.O_CREATE|os.O_WRONLY|os.O_EXCL, + 0600, + ) + if err != nil { + return fmt.Errorf(" failed to create %q, it already exists [%s]", filename, err) + } + f.Close() + + if err := clientcmd.WriteToFile(*kubeconfig, filename); err != nil { + return fmt.Errorf(" failed to write to %q [%s]", filename, err) + } + + fmt.Println(" created %q", filename) + return nil +} diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go new file mode 100644 index 00000000000..91f5f78d5fc --- /dev/null +++ b/pkg/kubeadm/util/tokens.go @@ -0,0 +1,91 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubeadmutil + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "strings" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +const ( + TokenIDLen = 6 + TokenBytes = 8 +) + +func randBytes(length int) ([]byte, string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return nil, "", err + } + // It's only the tokenID that doesn't care about raw byte slice, + // so we just encoded it in place and ignore bytes slice where we + // do not want it + return b, hex.EncodeToString(b), nil +} + +func GenerateToken(params *kubeadmapi.BootstrapParams) error { + _, tokenID, err := randBytes(TokenIDLen / 2) + if err != nil { + return err + } + + tokenBytes, token, err := randBytes(TokenBytes) + if err != nil { + return err + } + + params.Discovery.TokenID = tokenID + params.Discovery.BearerToken = token + params.Discovery.Token = tokenBytes + params.Discovery.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) + return nil +} + +func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { + if params.Discovery.GivenToken == "" { + return false, nil + } + givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".") + // TODO print desired format + // TODO could also print more specific messages in each case + invalidErr := " provided token is invalid - %s" + if len(givenToken) != 2 { + return false, fmt.Errorf(invalidErr, "not in 2-part dot-separated format") + } + if len(givenToken[0]) != TokenIDLen { + return false, fmt.Errorf(invalidErr, fmt.Sprintf( + "length of first part is incorrect [%d (given) != %d (expected) ]", + len(givenToken[0]), TokenIDLen)) + } + tokenBytes, err := hex.DecodeString(givenToken[1]) + if err != nil { + return false, fmt.Errorf(invalidErr, err) + } + if len(tokenBytes) != TokenBytes { + return false, fmt.Errorf(invalidErr, fmt.Sprintf( + "length of second part is incorrect [%d (given) != %d (expected)]", + len(tokenBytes), TokenBytes)) + } + params.Discovery.TokenID = givenToken[0] + params.Discovery.BearerToken = givenToken[1] + params.Discovery.Token = tokenBytes + return true, nil // given and valid +} From a82c4903150045b8ca60c30eaef0ad3ab05ba923 Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Thu, 1 Sep 2016 11:40:32 +0300 Subject: [PATCH 02/32] Fix compilation errors after method GetDefaultHostIP was removed --- pkg/kubeadm/cmd/init.go | 5 +++-- pkg/kubeadm/cmd/manual.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index 5c7a541900b..a40f32637a3 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -27,6 +27,7 @@ import ( kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + netutil "k8s.io/kubernetes/pkg/util/net" ) var ( @@ -59,11 +60,11 @@ func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Comman func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { if params.Discovery.ListenIP == "" { - ip, err := kubeadmutil.GetDefaultHostIP() + ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip + params.Discovery.ListenIP = ip.String() } if err := kubemaster.CreateTokenAuthFile(params); err != nil { return err diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index 05902986f66..aa617eba9f7 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -98,7 +98,7 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap if err != nil { return fmt.Errorf("Unable to autodetect IP address [%s], please specify with --listen-ip", err) } - params.Discovery.ListenIP = ip + params.Discovery.ListenIP = ip.String() } if err := kubemaster.CreateTokenAuthFile(params); err != nil { return err From 1c132fe974c563cdfb5c1bc70d6a7d4c14c9d806 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Fri, 2 Sep 2016 11:19:38 +0200 Subject: [PATCH 03/32] Address comments in review - start cleaning up `cmd/manual.go` - refine progress and error messages - add missing blank lines after the license headers - run `gofmt -s -w` - do not set fake cloud provider - add a note on why we cannot remove `HostNetwork: true` from `kube-discovery` pod just yet - taint master and use `role=master`, set tolerations and affinity for `kube-discovery` - parametrise log-level flag for all components --- cmd/kubeadm/app/kubeadm.go | 14 +-- pkg/kubeadm/api/types.go | 3 +- pkg/kubeadm/cmd/init.go | 4 + pkg/kubeadm/cmd/manual.go | 147 +++++++++++++++++--------------- pkg/kubeadm/master/addons.go | 4 +- pkg/kubeadm/master/apiclient.go | 99 +++++++++++++++++++-- pkg/kubeadm/master/discovery.go | 23 +++-- pkg/kubeadm/master/manifests.go | 10 +-- pkg/kubeadm/master/pki.go | 2 +- pkg/kubeadm/master/tokens.go | 3 + pkg/kubeadm/node/discovery.go | 3 +- pkg/kubeadm/util/kubeconfig.go | 2 +- pkg/kubeadm/util/tokens.go | 4 +- 13 files changed, 220 insertions(+), 98 deletions(-) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 584909353c4..239ef35dfb8 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path" + "runtime" "strings" "github.com/spf13/pflag" @@ -40,13 +41,16 @@ func getEnvParams() map[string]string { } envParams := map[string]string{ - "prefix": globalPrefix, - "host_pki_path": path.Join(globalPrefix, "pki"), - "hyperkube_image": "gcr.io/google_containers/hyperkube:v1.4.0-alpha.3", - "discovery_image": "dgoodwin/kubediscovery:latest", + "prefix": globalPrefix, + "host_pki_path": path.Join(globalPrefix, "pki"), + // TODO find a way to specify image versions for all of these... + "hyperkube_image": fmt.Sprintf("gcr.io/google_containers/hyperkube-%s:%s", runtime.GOARCH, "v1.4.0-alpha.3"), + "discovery_image": "dgoodwin/kubediscovery:latest", + "etcd_image": fmt.Sprintf("gcr.io/google_containers/etcd-%s:%s", runtime.GOARCH, "2.2.5"), + "component_loglevel": "--v=4", } - for k, _ := range envParams { + for k := range envParams { if v := os.Getenv(fmt.Sprintf("KUBE_%s", strings.ToUpper(k))); v != "" { envParams[k] = v } diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go index e659162cb0e..087a2d315d7 100644 --- a/pkg/kubeadm/api/types.go +++ b/pkg/kubeadm/api/types.go @@ -17,8 +17,7 @@ limitations under the License. package kubeadmapi type BootstrapParams struct { - // A struct with methods that implement Discover() - // kubeadm will do the CSR dance + // TODO this is mostly out of date and bloated now, let's revisit this soon Discovery *OutOfBandDiscovery EnvParams map[string]string } diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index a40f32637a3..abfc4647ab4 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -91,6 +91,10 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmap return err } + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil { + return err + } + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil { return err } diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index aa617eba9f7..491ab06b250 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -27,12 +27,12 @@ import ( kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" kubenode "k8s.io/kubernetes/pkg/kubeadm/node" kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" netutil "k8s.io/kubernetes/pkg/util/net" - // TODO: cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) var ( - manual_done_msgf = dedent.Dedent(` + manual_init_done_msgf = dedent.Dedent(` Master initialization complete: * Static pods written and kubelet's kubeconfig written. @@ -48,6 +48,14 @@ var ( kubeadm manual bootstrap join-node --ca-cert-file \ --token %s --api-server-urls https://%s:443/ `) + manual_join_done_msgf = dedent.Dedent(` + Node join complete: + * Certificate signing request sent to master and response + received. + * Kubelet informed of new secure connection details. + + Run 'kubectl get nodes' on the master to see this node join. + `) ) // TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken @@ -92,42 +100,9 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap Will create TLS certificates and set up static pods for Kubernetes master components. `), - RunE: func(cmd *cobra.Command, args []string) error { - if params.Discovery.ListenIP == "" { - ip, err := netutil.ChooseHostInterface() - if err != nil { - return fmt.Errorf("Unable to autodetect IP address [%s], please specify with --listen-ip", err) - } - params.Discovery.ListenIP = ip.String() - } - if err := kubemaster.CreateTokenAuthFile(params); err != nil { - return err - } - if err := kubemaster.WriteStaticPodManifests(params); err != nil { - return err - } - caKey, caCert, err := kubemaster.CreatePKIAssets(params) - if err != nil { - return err - } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) - if err != nil { - return err - } - for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { - out.Write([]byte(fmt.Sprintf("Unable to write admin for master:\n%s\n", err))) - return nil - } - } - - // TODO use templates to reference struct fields directly as order of args is fragile - fmt.Fprintf(out, manual_done_msgf, - params.Discovery.BearerToken, - params.Discovery.ListenIP, - ) - - return nil + Run: func(cmd *cobra.Command, args []string) { + err := RunManualBootstrapInitMaster(out, cmd, args, params) + cmdutil.CheckErr(err) }, } @@ -143,43 +118,50 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap return cmd } +func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + if params.Discovery.ListenIP == "" { + ip, err := netutil.ChooseHostInterface() + if err != nil { + return fmt.Errorf(" unable to autodetect IP address [%s], please specify with --listen-ip", err) + } + params.Discovery.ListenIP = ip.String() + } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { + return err + } + if err := kubemaster.WriteStaticPodManifests(params); err != nil { + return err + } + caKey, caCert, err := kubemaster.CreatePKIAssets(params) + if err != nil { + return err + } + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + if err != nil { + return err + } + for name, kubeconfig := range kubeconfigs { + if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + return err + } + } + + // TODO use templates to reference struct fields directly as order of args is fragile + fmt.Fprintf(out, manual_init_done_msgf, + params.Discovery.BearerToken, + params.Discovery.ListenIP, + ) + return nil +} + func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { cmd := &cobra.Command{ Use: "join-node", Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", Run: func(cmd *cobra.Command, args []string) { - if params.Discovery.CaCertFile == "" { - out.Write([]byte(fmt.Sprintf("Must specify --ca-cert-file (see --help)\n"))) - return - } - - if params.Discovery.ApiServerURLs == "" { - out.Write([]byte(fmt.Sprintf("Must specify --api-server-urls (see --help)\n"))) - return - } - - kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) - if err != nil { - out.Write([]byte(fmt.Sprintf("Failed to perform TLS bootstrap: %s\n", err))) - return - } - //fmt.Println("recieved signed certificate from the API server, will write `/etc/kubernetes/kubelet.conf`...") - - err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) - if err != nil { - out.Write([]byte(fmt.Sprintf("Unable to write config for node:\n%s\n", err))) - return - } - out.Write([]byte(dedent.Dedent(` - Node join complete: - * Certificate signing request sent to master and response - received. - * Kubelet informed of new secure connection details. - - Run 'kubectl get nodes' on the master to see this node join. - - `))) + err := RunManualBootstrapJoinNode(out, cmd, args, params) + cmdutil.CheckErr(err) }, } cmd.PersistentFlags().StringVarP(¶ms.Discovery.CaCertFile, "ca-cert-file", "", "", @@ -193,3 +175,30 @@ func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapPa return cmd } + +func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + if params.Discovery.CaCertFile == "" { + fmt.Fprintf(out, "Must specify --ca-cert-file (see --help)\n") + return nil + } + + if params.Discovery.ApiServerURLs == "" { + fmt.Fprintf(out, "Must specify --api-server-urls (see --help)\n") + return nil + } + + kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) + if err != nil { + fmt.Fprintf(out, "Failed to perform TLS bootstrap: %s\n", err) + return err + } + + err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + if err != nil { + fmt.Fprintf(out, "Unable to write config for node:\n%s\n", err) + return err + } + + fmt.Fprintf(out, manual_join_done_msgf) + return nil +} diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index 97911ede831..e56077ceff8 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package kubemaster import ( @@ -35,7 +36,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { "/hyperkube", "proxy", "--kubeconfig=/run/kubeconfig", - COMPONENT_LOGLEVEL, + params.EnvParams["component_loglevel"], }, SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, VolumeMounts: []api.VolumeMount{ @@ -80,6 +81,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) + SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go index 4a2d404d9bf..6c9b19c721e 100644 --- a/pkg/kubeadm/master/apiclient.go +++ b/pkg/kubeadm/master/apiclient.go @@ -17,10 +17,12 @@ limitations under the License. package kubemaster import ( + "encoding/json" "fmt" "time" "k8s.io/kubernetes/pkg/api" + apierrs "k8s.io/kubernetes/pkg/api/errors" unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" @@ -53,6 +55,7 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli if err != nil { return false, nil } + // TODO revisit this when we implement HA if len(cs.Items) < 3 { fmt.Println(" not all control plane components are ready yet") return false, nil @@ -66,11 +69,32 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli } } - fmt.Printf(" all control plane components are healthy after %s seconds\n", time.Since(start).Seconds()) + fmt.Printf(" all control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) + return true, nil + }) + + fmt.Println(" waiting for at least one node to register and become ready") + start = time.Now() + wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + nodeList, err := client.Nodes().List(api.ListOptions{}) + if err != nil { + fmt.Println(" temporarily unable to list nodes (will retry)") + return false, nil + } + if len(nodeList.Items) < 1 { + //fmt.Printf(" %d nodes have registered so far", len(nodeList.Items)) + return false, nil + } + n := &nodeList.Items[0] + if !api.IsNodeReady(n) { + fmt.Println(" first node has registered, but is not ready yet") + return false, nil + } + + fmt.Printf(" first node is ready after %f seconds\n", time.Since(start).Seconds()) return true, nil }) - // TODO may be also check node status return client, nil } @@ -103,9 +127,72 @@ func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) * } } -func TaintMaster(*clientset.Clientset) error { - // TODO - annotations := make(map[string]string) - annotations[api.TaintsAnnotationKey] = "" +// It's safe to do this for alpha, as we don't have HA and there is no way we can get +// more then one node here (TODO find a way to determine owr own node name) +func findMyself(client *clientset.Clientset) (*api.Node, error) { + nodeList, err := client.Nodes().List(api.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to list nodes [%s]", err) + } + if len(nodeList.Items) < 1 { + return nil, fmt.Errorf("no nodes found") + } + node := &nodeList.Items[0] + return node, nil +} + +func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error { + n, err := findMyself(client) + if err != nil { + return err + } + + n.ObjectMeta.Labels["kubeadm.alpha.kubernetes.io/role"] = "master" + taintsAnnotation, _ := json.Marshal([]api.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) + n.ObjectMeta.Annotations[api.TaintsAnnotationKey] = string(taintsAnnotation) + + if _, err := client.Nodes().Update(n); err != nil { + if apierrs.IsConflict(err) { + fmt.Println(" temporarily unable to update master node metadata due to conflict (will retry)") + time.Sleep(500 * time.Millisecond) + attemptToUpdateMasterRoleLabelsAndTaints(client) + } else { + return err + } + } + return nil } + +func UpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error { + err := attemptToUpdateMasterRoleLabelsAndTaints(client) + if err != nil { + return fmt.Errorf(" failed to update master node - %s", err) + } + return nil +} + +func SetMasterTaintTolerations(meta *api.ObjectMeta) { + tolerationsAnnotation, _ := json.Marshal([]api.Toleration{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) + if meta.Annotations == nil { + meta.Annotations = map[string]string{} + } + meta.Annotations[api.TolerationsAnnotationKey] = string(tolerationsAnnotation) +} + +func SetMasterNodeAffinity(meta *api.ObjectMeta) { + nodeAffinity := &api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{{ + MatchExpressions: []api.NodeSelectorRequirement{{ + Key: "kubeadm.alpha.kubernetes.io/role", Operator: api.NodeSelectorOpIn, Values: []string{"master"}, + }}, + }}, + }, + } + affinityAnnotation, _ := json.Marshal(api.Affinity{NodeAffinity: nodeAffinity}) + if meta.Annotations == nil { + meta.Annotations = map[string]string{} + } + meta.Annotations[api.AffinityAnnotationKey] = string(affinityAnnotation) +} diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go index 76caac90090..51fc65bde8e 100644 --- a/pkg/kubeadm/master/discovery.go +++ b/pkg/kubeadm/master/discovery.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package kubemaster import ( @@ -61,7 +62,11 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { return api.PodSpec{ - SecurityContext: &api.PodSecurityContext{HostNetwork: true}, // TODO we should just use map it to a host port + // We have to use host network namespace, as `HostPort`/`HostIP` are Docker's + // buisness and CNI support isn't quite there yet (except for kubenet) + // (see https://github.com/kubernetes/kubernetes/issues/31307) + // TODO update this when #31307 is resolved + SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ Name: kubeDiscoverynName, Image: params.EnvParams["discovery_image"], @@ -71,6 +76,10 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { MountPath: "/tmp/secret", // TODO use a shared constant ReadOnly: true, }}, + Ports: []api.ContainerPort{ + // TODO when CNI issue (#31307) is resolved, we should add `HostIP: params.Discovery.ListenIP` + {Name: "http", ContainerPort: 9898, HostPort: 9898}, + }, }}, Volumes: []api.Volume{{ Name: kubeDiscoverySecretName, @@ -82,8 +91,7 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery { - // TODO pin to master - return kubeDiscovery{ + kd := kubeDiscovery{ Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)), Secret: &api.Secret{ ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, @@ -91,16 +99,21 @@ func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certifica Data: encodeKubeDiscoverySecretData(params, caCert), }, } + + SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta) + SetMasterNodeAffinity(&kd.Deployment.Spec.Template.ObjectMeta) + + return kd } func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error { kd := newKubeDiscovery(params, caCert) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { - return fmt.Errorf(" failed to create %q deployment", kubeDiscoverynName) + return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoverynName, err) } if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil { - return fmt.Errorf(" failed to create %q secret", kubeDiscoverySecretName) + return fmt.Errorf(" failed to create %q secret [%s]", kubeDiscoverySecretName, err) } fmt.Println(" created essential addon: kube-discovery") diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index b8810be2b6f..7641ee10661 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -35,7 +35,6 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - COMPONENT_LOGLEVEL = "--v=4" SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=10.16.0.0/12" CLUSTER_NAME = "--cluster-name=kubernetes" MASTER = "--master=127.0.0.1:8080" @@ -55,7 +54,7 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "--advertise-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", "--data-dir=/var/etcd/data", }, - Image: "gcr.io/google_containers/etcd:2.2.1", // TODO parametrise + Image: params.EnvParams["etcd_image"], LivenessProbe: componentProbe(2379, "/health"), Name: "etcd-server", Resources: componentResources("200m"), @@ -69,7 +68,6 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "apiserver", "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", - "--cloud-provider=fake", // TODO parametrise "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", SERVICE_CLUSTER_IP_RANGE, "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", @@ -78,7 +76,7 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", "--secure-port=443", "--allow-privileged", - COMPONENT_LOGLEVEL, + params.EnvParams["component_loglevel"], "--token-auth-file=/etc/kubernetes/pki/tokens.csv", }, VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, @@ -99,7 +97,7 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", - COMPONENT_LOGLEVEL, + params.EnvParams["component_loglevel"], }, VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), @@ -113,7 +111,7 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "scheduler", "--leader-elect", MASTER, - COMPONENT_LOGLEVEL, + params.EnvParams["component_loglevel"], }, LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index 6c23c90a5de..d5b24349855 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -175,6 +175,6 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 } // TODO print a summary of SANs used and checksums (signatures) of each of the certiicates - fmt.Println(" created keys and certificates in %q", params.EnvParams["host_pki_path"]) + fmt.Printf(" created keys and certificates in %q\n", params.EnvParams["host_pki_path"]) return caKey, caCert, nil } diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index 81d899807b0..ad184cbbcd2 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package kubemaster import ( @@ -38,6 +39,8 @@ func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { return err } fmt.Printf(" generated token: %q\n", params.Discovery.GivenToken) + } else { + fmt.Println(" accepted provided token") } return nil diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index 9609905b65e..beb470e98df 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package kubenode import ( @@ -43,7 +44,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) } - fmt.Println(" created cluster info discovery client, requesting info from %q", requestURL) + fmt.Printf(" created cluster info discovery client, requesting info from %q\n", requestURL) res, err := http.DefaultClient.Do(req) if err != nil { diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 9c595d57580..49e27e3f29a 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -98,6 +98,6 @@ func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, return fmt.Errorf(" failed to write to %q [%s]", filename, err) } - fmt.Println(" created %q", filename) + fmt.Printf(" created %q\n", filename) return nil } diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 91f5f78d5fc..5d643dcf6e1 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package kubeadmutil import ( @@ -61,8 +62,9 @@ func GenerateToken(params *kubeadmapi.BootstrapParams) error { func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { if params.Discovery.GivenToken == "" { - return false, nil + return false, nil // not given } + fmt.Println(" validating provided token") givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".") // TODO print desired format // TODO could also print more specific messages in each case From 9e4fc59d396373249fb9a360a663cf65f9ed2030 Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Mon, 5 Sep 2016 18:41:34 +0100 Subject: [PATCH 04/32] Added DNS add-on. --- cmd/kubeadm/app/kubeadm.go | 2 + pkg/kubeadm/master/addons.go | 164 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 239ef35dfb8..8631debd5b1 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -48,6 +48,8 @@ func getEnvParams() map[string]string { "discovery_image": "dgoodwin/kubediscovery:latest", "etcd_image": fmt.Sprintf("gcr.io/google_containers/etcd-%s:%s", runtime.GOARCH, "2.2.5"), "component_loglevel": "--v=4", + "dns_domain": "cluster.local", + "dns_replicas": "1", } for k := range envParams { diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index e56077ceff8..bae44d7c434 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -19,10 +19,15 @@ package kubemaster import ( "fmt" "path" + "strconv" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + "k8s.io/kubernetes/pkg/util/intstr" ) func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { @@ -79,6 +84,158 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } +func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Deployment { + metaLabels := map[string]string{ + "k8s-app": "kube-dns", + "version": "v19", + "kubernetes.io/cluster-service": "true", + } + + metaAnnotations := map[string]string{ + "scheduler.alpha.kubernetes.io/critical-pod": "''", + "scheduler.alpha.kubernetes.io/tolerations": "'[{\"key\":\"CriticalAddonsOnly\", \"operator\":\"Exists\"}]'", + } + + dnsPodResources := api.ResourceList{ + api.ResourceLimitsCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("170Mi"), + } + + healthzPodResources := api.ResourceList{ + api.ResourceLimitsCPU: resource.MustParse("10m"), + api.ResourceMemory: resource.MustParse("50Mi"), + } + + podSpec := api.PodSpec{ + Containers: []api.Container{ + // DNS server + { + Name: "kube-dns", + Image: "gcr.io/google_containers/kubedns-amd64:1.7", + Resources: api.ResourceRequirements{ + Limits: dnsPodResources, + Requests: dnsPodResources, + }, + Args: []string{ + "--domain=" + params.EnvParams["dns_domain"], + "--dns-port=10053", + // TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__ + }, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + Scheme: api.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + TimeoutSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 1, + }, + // # we poll on pod startup for the Kubernetes master service and + // # only setup the /readiness HTTP server once that's available. + ReadinessProbe: &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Path: "/readiness", + Port: intstr.FromInt(8081), + Scheme: api.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + Ports: []api.ContainerPort{ + { + ContainerPort: 10053, + Name: "dns-local", + Protocol: api.ProtocolUDP, + }, + { + ContainerPort: 10053, + Name: "dns-tcp-local", + Protocol: api.ProtocolTCP, + }, + }, + }, + // dnsmasq + { + Name: "dnsmasq", + Image: "gcr.io/google_containers/kube-dnsmasq-amd64:1.3", + Resources: api.ResourceRequirements{ + Limits: dnsPodResources, + Requests: dnsPodResources, + }, + Args: []string{ + "--cache-size=1000", + "--no-resolv", + "--server=127.0.0.1#10053", + }, + Ports: []api.ContainerPort{ + { + ContainerPort: 53, + Name: "dns", + Protocol: api.ProtocolUDP, + }, + { + ContainerPort: 53, + Name: "dns-tcp", + Protocol: api.ProtocolTCP, + }, + }, + }, + // healthz + { + Name: "healthz", + Image: "gcr.io/google_containers/exechealthz-amd64:1.1", + Resources: api.ResourceRequirements{ + Limits: healthzPodResources, + Requests: healthzPodResources, + }, + Args: []string{ + "-cmd=nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1:10053 >/dev/null", + "-port=8080", + "-quiet", + }, + Ports: []api.ContainerPort{ + { + ContainerPort: 8080, + Protocol: api.ProtocolTCP, + }, + }, + }, + }, + DNSPolicy: api.DNSDefault, + } + + dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) + if err != nil { + dnsReplicas = 1 + } + + return &extensions.Deployment{ + ObjectMeta: api.ObjectMeta{ + Name: "kube-dns-v19", + Namespace: "kube-system", + Labels: metaLabels, + }, + Spec: extensions.DeploymentSpec{ + Replicas: int32(dnsReplicas), + Selector: &unversionedapi.LabelSelector{MatchLabels: metaLabels}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{ + Labels: metaLabels, + Annotations: metaAnnotations, + }, + Spec: podSpec, + }, + }, + } + +} + func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) @@ -91,5 +248,12 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset // TODO should we wait for it to become ready at least on the master? + kubeDnsDeployment := createKubeDnsDeployment(params) + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDnsDeployment); err != nil { + return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) + } + + fmt.Println(" created essential addon: kube-dns") + return nil } From b48df06aba04f3e74e7ab38373ef7bd7c469a7ae Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 6 Sep 2016 16:30:58 +0200 Subject: [PATCH 05/32] Refactor kube-dns addon constructors, more labels - also add another IP to SANs - fix mkdir calls - add TODO for ComponentConfig - start tagging TODOs by phases --- cmd/kubeadm/app/kubeadm.go | 5 +- pkg/kubeadm/cmd/cmd.go | 10 ++-- pkg/kubeadm/master/addons.go | 88 ++++++++++++++------------------- pkg/kubeadm/master/apiclient.go | 22 ++++++++- pkg/kubeadm/master/manifests.go | 2 +- pkg/kubeadm/master/pki.go | 2 +- pkg/kubeadm/master/tokens.go | 2 +- pkg/kubeadm/node/discovery.go | 8 +-- pkg/kubeadm/util/kubeconfig.go | 4 ++ 9 files changed, 75 insertions(+), 68 deletions(-) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 8631debd5b1..b7249e995b3 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -41,9 +41,8 @@ func getEnvParams() map[string]string { } envParams := map[string]string{ - "prefix": globalPrefix, - "host_pki_path": path.Join(globalPrefix, "pki"), - // TODO find a way to specify image versions for all of these... + "prefix": globalPrefix, + "host_pki_path": path.Join(globalPrefix, "pki"), "hyperkube_image": fmt.Sprintf("gcr.io/google_containers/hyperkube-%s:%s", runtime.GOARCH, "v1.4.0-alpha.3"), "discovery_image": "dgoodwin/kubediscovery:latest", "etcd_image": fmt.Sprintf("gcr.io/google_containers/etcd-%s:%s", runtime.GOARCH, "2.2.5"), diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index 2fe3ae0f169..c152f768ec6 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -65,22 +65,22 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env `), } - // TODO figure out how to avoid running as root + // TODO(phase2+) figure out how to avoid running as root // - // TODO also print the alpha warning when running any commands, as well as + // TODO(phase1) also print the alpha warning when running any commands, as well as // in the help text. // - // TODO detect interactive vs non-interactive use and adjust output accordingly + // TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly // i.e. make it automation friendly // - // TODO create an bastraction that defines files and the content that needs to + // TODO(phase2) create an bastraction that defines files and the content that needs to // be written to disc and write it all in one go at the end as we have a lot of // crapy little files written from different parts of this code; this could also // be useful for testing bootstrapParams := &kubeadmapi.BootstrapParams{ Discovery: &kubeadmapi.OutOfBandDiscovery{ - // TODO this type no longer makes sense here + // TODO(phase1) this type no longer makes sense here }, EnvParams: envParams, } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index bae44d7c434..a37ba373d4d 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -19,12 +19,11 @@ package kubemaster import ( "fmt" "path" + "runtime" "strconv" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" - unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" "k8s.io/kubernetes/pkg/util/intstr" @@ -84,34 +83,24 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Deployment { - metaLabels := map[string]string{ - "k8s-app": "kube-dns", - "version": "v19", - "kubernetes.io/cluster-service": "true", - } - - metaAnnotations := map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "''", - "scheduler.alpha.kubernetes.io/tolerations": "'[{\"key\":\"CriticalAddonsOnly\", \"operator\":\"Exists\"}]'", - } +func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { dnsPodResources := api.ResourceList{ - api.ResourceLimitsCPU: resource.MustParse("100m"), - api.ResourceMemory: resource.MustParse("170Mi"), + api.ResourceName(api.ResourceCPU): resource.MustParse("100m"), + api.ResourceName(api.ResourceMemory): resource.MustParse("170Mi"), } healthzPodResources := api.ResourceList{ - api.ResourceLimitsCPU: resource.MustParse("10m"), - api.ResourceMemory: resource.MustParse("50Mi"), + api.ResourceName(api.ResourceCPU): resource.MustParse("10m"), + api.ResourceName(api.ResourceMemory): resource.MustParse("50Mi"), } - podSpec := api.PodSpec{ + return api.PodSpec{ Containers: []api.Container{ // DNS server { Name: "kube-dns", - Image: "gcr.io/google_containers/kubedns-amd64:1.7", + Image: "gcr.io/google_containers/kubedns-" + runtime.GOARCH + ":1.7", Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -163,7 +152,7 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep // dnsmasq { Name: "dnsmasq", - Image: "gcr.io/google_containers/kube-dnsmasq-amd64:1.3", + Image: "gcr.io/google_containers/kube-dnsmasq-" + runtime.GOARCH + ":1.3", Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -189,7 +178,7 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep // healthz { Name: "healthz", - Image: "gcr.io/google_containers/exechealthz-amd64:1.1", + Image: "gcr.io/google_containers/exechealthz-" + runtime.GOARCH + ":1.1", Resources: api.ResourceRequirements{ Limits: healthzPodResources, Requests: healthzPodResources, @@ -199,41 +188,25 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep "-port=8080", "-quiet", }, - Ports: []api.ContainerPort{ - { - ContainerPort: 8080, - Protocol: api.ProtocolTCP, - }, - }, + Ports: []api.ContainerPort{{ + ContainerPort: 8080, + Protocol: api.ProtocolTCP, + }}, }, }, DNSPolicy: api.DNSDefault, } - dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) - if err != nil { - dnsReplicas = 1 - } - - return &extensions.Deployment{ - ObjectMeta: api.ObjectMeta{ - Name: "kube-dns-v19", - Namespace: "kube-system", - Labels: metaLabels, - }, - Spec: extensions.DeploymentSpec{ - Replicas: int32(dnsReplicas), - Selector: &unversionedapi.LabelSelector{MatchLabels: metaLabels}, - Template: api.PodTemplateSpec{ - ObjectMeta: api.ObjectMeta{ - Labels: metaLabels, - Annotations: metaAnnotations, - }, - Spec: podSpec, - }, +} +func createKubeDNSServiceSpec(params *kubeadmapi.BootstrapParams) api.ServiceSpec { + return api.ServiceSpec{ + Selector: map[string]string{"name": "kube-dns"}, + Ports: []api.ServicePort{ + {Name: "dns", Port: 53, Protocol: api.ProtocolUDP}, + {Name: "dns-tcp", Port: 53, Protocol: api.ProtocolTCP}, }, + ClusterIP: "100.64.0.2", } - } func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { @@ -248,8 +221,21 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset // TODO should we wait for it to become ready at least on the master? - kubeDnsDeployment := createKubeDnsDeployment(params) - if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDnsDeployment); err != nil { + dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) + if err != nil { + dnsReplicas = 1 + } + + kubeDNSDeployment := NewDeployment("kube-dns", int32(dnsReplicas), createKubeDNSPodSpec(params)) + SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) + + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { + return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) + } + + kubeDNSService := NewService("kube-dns", createKubeDNSServiceSpec(params)) + + if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go index 6c9b19c721e..b2e22c895e3 100644 --- a/pkg/kubeadm/master/apiclient.go +++ b/pkg/kubeadm/master/apiclient.go @@ -98,8 +98,15 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli return client, nil } +func standardLabels(n string) map[string]string { + return map[string]string{ + "component": n, "name": n, "k8s-app": n, + "kubernetes.io/cluster-service": "true", "tier": "node", + } +} + func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet { - l := map[string]string{"component": daemonName, "tier": "node"} + l := standardLabels(daemonName) return &extensions.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: daemonName}, Spec: extensions.DaemonSetSpec{ @@ -112,8 +119,19 @@ func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet } } +func NewService(serviceName string, spec api.ServiceSpec) *api.Service { + l := standardLabels(serviceName) + return &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: serviceName, + Labels: l, + }, + Spec: spec, + } +} + func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) *extensions.Deployment { - l := map[string]string{"name": deploymentName} + l := standardLabels(deploymentName) return &extensions.Deployment{ ObjectMeta: api.ObjectMeta{Name: deploymentName}, Spec: extensions.DeploymentSpec{ diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 7641ee10661..b104889655a 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -35,7 +35,7 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=10.16.0.0/12" + SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=100.64.0.0/12" CLUSTER_NAME = "--cluster-name=kubernetes" MASTER = "--master=127.0.0.1:8080" ) diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index d5b24349855..e908c65f6c6 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -57,7 +57,7 @@ func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNam return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) } // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly - altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1")) + altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"), net.ParseIP("10.16.0.1"), net.ParseIP("100.64.0.1")) altNames.DNSNames = append(altNames.DNSNames, "kubernetes", "kubernetes.default", diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index ad184cbbcd2..ff052575a77 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -51,7 +51,7 @@ func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { if err := generateTokenIfNeeded(params); err != nil { return fmt.Errorf(" failed to generate token(s) [%s]", err) } - if err := os.MkdirAll(path.Join(params.EnvParams["host_pki_path"]), 0700); err != nil { + if err := os.MkdirAll(params.EnvParams["host_pki_path"], 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) } serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index beb470e98df..3b41cdf0b27 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -59,17 +59,17 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda return nil, fmt.Errorf(" failed to parse response as JWS object [%s]", err) } - fmt.Println(" cluster info object recieved, verifying signature using given token") + fmt.Println(" cluster info object received, verifying signature using given token") output, err := object.Verify(params.Discovery.Token) if err != nil { - return nil, fmt.Errorf(" failed to verify JWS signature of recieved cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%s]", err) } clusterInfo := kubeadmapi.ClusterInfo{} if err := json.Unmarshal(output, &clusterInfo); err != nil { - return nil, fmt.Errorf(" failed to unmarshal recieved cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to decode received cluster info object [%s]", err) } if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 { @@ -77,7 +77,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda } // TODO print checksum of the CA certificate - fmt.Printf(" cluser info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) + fmt.Printf(" cluster info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) // TODO we need to configure the client to validate the server // if it is signed by any of the returned certificates diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 49e27e3f29a..041021bf9cb 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -82,6 +82,10 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // start it again in that case). func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { + if err := os.MkdirAll(params.EnvParams["prefix"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["prefix"], err) + } + filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile( From 26c4f593aa8b1bc67351adef3336da5f4ff98532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Wed, 7 Sep 2016 15:53:11 +0300 Subject: [PATCH 06/32] Cleanup/refactor some things, make it possible to use individual images, hide unused flags --- cmd/kubeadm/app/kubeadm.go | 12 ++--- cmd/kubeadm/kubeadm.go | 1 + pkg/kubeadm/api/types.go | 9 +++- pkg/kubeadm/cmd/init.go | 14 +++--- pkg/kubeadm/cmd/manual.go | 14 +++--- pkg/kubeadm/images/images.go | 76 ++++++++++++++++++++++++++++++++ pkg/kubeadm/master/addons.go | 29 +++++------- pkg/kubeadm/master/apiclient.go | 2 +- pkg/kubeadm/master/discovery.go | 2 +- pkg/kubeadm/master/kubeconfig.go | 2 +- pkg/kubeadm/master/manifests.go | 42 ++++++++++-------- pkg/kubeadm/master/pki.go | 6 +-- pkg/kubeadm/master/tokens.go | 2 +- pkg/kubeadm/node/csr.go | 2 +- pkg/kubeadm/node/discovery.go | 4 +- pkg/kubeadm/util/kubeconfig.go | 2 +- pkg/kubeadm/util/tokens.go | 2 +- 17 files changed, 153 insertions(+), 68 deletions(-) create mode 100644 pkg/kubeadm/images/images.go diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index b7249e995b3..749948d5c34 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "path" - "runtime" "strings" "github.com/spf13/pflag" @@ -43,12 +42,11 @@ func getEnvParams() map[string]string { envParams := map[string]string{ "prefix": globalPrefix, "host_pki_path": path.Join(globalPrefix, "pki"), - "hyperkube_image": fmt.Sprintf("gcr.io/google_containers/hyperkube-%s:%s", runtime.GOARCH, "v1.4.0-alpha.3"), - "discovery_image": "dgoodwin/kubediscovery:latest", - "etcd_image": fmt.Sprintf("gcr.io/google_containers/etcd-%s:%s", runtime.GOARCH, "2.2.5"), + "hyperkube_image": "", + "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), + "etcd_image": "", "component_loglevel": "--v=4", "dns_domain": "cluster.local", - "dns_replicas": "1", } for k := range envParams { @@ -65,6 +63,10 @@ func Run() error { logs.InitLogs() defer logs.FlushLogs() + // We do not want these flags to show up in --help + pflag.CommandLine.MarkHidden("google-json-key") + pflag.CommandLine.MarkHidden("log-flush-frequency") + cmd := cmd.NewKubeadmCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr, getEnvParams()) return cmd.Execute() } diff --git a/cmd/kubeadm/kubeadm.go b/cmd/kubeadm/kubeadm.go index 6aca5a18f3d..f61297fc8ff 100644 --- a/cmd/kubeadm/kubeadm.go +++ b/cmd/kubeadm/kubeadm.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app" ) +// TODO(phase1): check for root func main() { if err := app.Run(); err != nil { os.Exit(1) diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go index 087a2d315d7..dab8765e871 100644 --- a/pkg/kubeadm/api/types.go +++ b/pkg/kubeadm/api/types.go @@ -14,7 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadmapi +package api + +import ( + "net" +) type BootstrapParams struct { // TODO this is mostly out of date and bloated now, let's revisit this soon @@ -32,7 +36,8 @@ type OutOfBandDiscovery struct { BearerToken string // set based on Token // 'init-master' side ApiServerDNSName string // optional, used in master bootstrap - ListenIP string // optional IP for master to listen on, rather than autodetect + ListenIP net.IP // optional IP for master to listen on, rather than autodetect + UseHyperkubeImage bool } type ClusterInfo struct { diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index abfc4647ab4..56422cf6439 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -42,7 +42,7 @@ var ( func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { cmd := &cobra.Command{ - Use: "init --token [--listen-ip ]", + Use: "init", Short: "Run this on the first server you deploy onto.", Run: func(cmd *cobra.Command, args []string) { err := RunInit(out, cmd, args, params) @@ -50,21 +50,25 @@ func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Comman }, } - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", + cmd.PersistentFlags().StringVar(¶ms.Discovery.GivenToken, "token", "", `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + cmd.PersistentFlags().BoolVar(¶ms.Discovery.UseHyperkubeImage, "use-hyperkube", false, + `(optional) Use the hyperkube image for running the apiserver, controller-manager, scheduler and proxy.`) return cmd } func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - if params.Discovery.ListenIP == "" { + + // Auto-detect the IP + if params.Discovery.ListenIP == nil { ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip.String() + params.Discovery.ListenIP = ip } if err := kubemaster.CreateTokenAuthFile(params); err != nil { return err diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index 491ab06b250..49a611ced12 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -107,25 +107,27 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap } params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", "", + cmd.PersistentFlags().StringVar(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", `(optional) DNS name for the API server, will be encoded into subjectAltName in the resulting (generated) TLS certificates`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + cmd.PersistentFlags().StringVar(¶ms.Discovery.BearerToken, "token", "", `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) return cmd } func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - if params.Discovery.ListenIP == "" { + // Auto-detect the IP + if params.Discovery.ListenIP == nil { ip, err := netutil.ChooseHostInterface() if err != nil { - return fmt.Errorf(" unable to autodetect IP address [%s], please specify with --listen-ip", err) + return err } - params.Discovery.ListenIP = ip.String() + params.Discovery.ListenIP = ip } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { return err } diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go new file mode 100644 index 00000000000..27f3fb03a60 --- /dev/null +++ b/pkg/kubeadm/images/images.go @@ -0,0 +1,76 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package images + +import ( + "fmt" + "runtime" +) + +const ( + KubeEtcdImage = "etcd" + + KubeApiServerImage = "apiserver" + KubeControllerManagerImage = "controller-manager" + KubeSchedulerImage = "scheduler" + KubeProxyImage = "proxy" + + KubeDnsImage = "kube-dns" + KubeDnsmasqImage = "dnsmasq" + KubeExechealthzImage = "exechealthz" + + + gcrPrefix = "gcr.io/google_containers" + etcdVersion = "2.2.5" + kubeVersion = "v1.4.0-alpha.3" + + kubeDnsVersion = "1.7" + dnsmasqVersion = "1.3" + exechealthzVersion = "1.1" +) + +func GetCoreImage(image string, overrideImage string, useHyperkube bool) string { + if overrideImage != "" { + return overrideImage + } + + if useHyperkube { + return map[string]string{ + KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), + KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), + }[image] + } + + return map[string]string{ + KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), + KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), + }[image] +} + +func GetAddonImage(image string) string { + return map[string]string{ + KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dns", runtime.GOARCH, kubeDnsVersion), + KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), + KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), + }[image] +} diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index a37ba373d4d..add2674d3fd 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -14,18 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "fmt" "path" - "runtime" - "strconv" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + "k8s.io/kubernetes/pkg/kubeadm/images" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -35,13 +34,11 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ Name: "kube-proxy", - Image: params.EnvParams["hyperkube_image"], - Command: []string{ - "/hyperkube", - "proxy", + Image: images.GetCoreImage(images.KubeProxyImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), + Command: append(getImageEntrypoint("proxy", params.Discovery.UseHyperkubeImage), []string{ "--kubeconfig=/run/kubeconfig", params.EnvParams["component_loglevel"], - }, + }...), SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, VolumeMounts: []api.VolumeMount{ { @@ -83,6 +80,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } +// TODO(phase1): Create the DNS service as well func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { dnsPodResources := api.ResourceList{ @@ -100,7 +98,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { // DNS server { Name: "kube-dns", - Image: "gcr.io/google_containers/kubedns-" + runtime.GOARCH + ":1.7", + Image: images.GetAddonImage(images.KubeDnsImage), Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -152,7 +150,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { // dnsmasq { Name: "dnsmasq", - Image: "gcr.io/google_containers/kube-dnsmasq-" + runtime.GOARCH + ":1.3", + Image: images.GetAddonImage(images.KubeDnsmasqImage), Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -178,7 +176,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { // healthz { Name: "healthz", - Image: "gcr.io/google_containers/exechealthz-" + runtime.GOARCH + ":1.1", + Image: images.GetAddonImage(images.KubeExechealthzImage), Resources: api.ResourceRequirements{ Limits: healthzPodResources, Requests: healthzPodResources, @@ -219,14 +217,7 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset fmt.Println(" created essential addon: kube-proxy") - // TODO should we wait for it to become ready at least on the master? - - dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) - if err != nil { - dnsReplicas = 1 - } - - kubeDNSDeployment := NewDeployment("kube-dns", int32(dnsReplicas), createKubeDNSPodSpec(params)) + kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(params)) SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go index b2e22c895e3..97654cbab52 100644 --- a/pkg/kubeadm/master/apiclient.go +++ b/pkg/kubeadm/master/apiclient.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "encoding/json" diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go index 51fc65bde8e..15ae71e9969 100644 --- a/pkg/kubeadm/master/discovery.go +++ b/pkg/kubeadm/master/discovery.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "crypto/x509" diff --git a/pkg/kubeadm/master/kubeconfig.go b/pkg/kubeadm/master/kubeconfig.go index 9cd9506310b..ea76c3849b6 100644 --- a/pkg/kubeadm/master/kubeconfig.go +++ b/pkg/kubeadm/master/kubeconfig.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "crypto/rsa" diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index b104889655a..385813dffe9 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "bytes" @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/api/unversioned" api "k8s.io/kubernetes/pkg/api/v1" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + "k8s.io/kubernetes/pkg/kubeadm/images" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -50,11 +51,11 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "etcd": componentPod(api.Container{ Command: []string{ "/usr/local/bin/etcd", - "--listen-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", - "--advertise-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", "--data-dir=/var/etcd/data", }, - Image: params.EnvParams["etcd_image"], + Image: images.GetCoreImage(images.KubeEtcdImage, params.EnvParams["etcd_image"], params.Discovery.UseHyperkubeImage), LivenessProbe: componentProbe(2379, "/health"), Name: "etcd-server", Resources: componentResources("200m"), @@ -62,10 +63,8 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { // TODO bind-mount certs in "kube-apiserver": componentPod(api.Container{ Name: "kube-apiserver", - Image: params.EnvParams["hyperkube_image"], - Command: []string{ - "/hyperkube", - "apiserver", + Image: images.GetCoreImage(images.KubeApiServerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), + Command: append(getImageEntrypoint("apiserver", params.Discovery.UseHyperkubeImage), []string{ "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", @@ -78,17 +77,15 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "--allow-privileged", params.EnvParams["component_loglevel"], "--token-auth-file=/etc/kubernetes/pki/tokens.csv", - }, + }...), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), }, pkiVolume(params)), "kube-controller-manager": componentPod(api.Container{ Name: "kube-controller-manager", - Image: params.EnvParams["hyperkube_image"], - Command: []string{ - "/hyperkube", - "controller-manager", + Image: images.GetCoreImage(images.KubeControllerManagerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), + Command: append(getImageEntrypoint("controller-manager", params.Discovery.UseHyperkubeImage), []string{ "--leader-elect", MASTER, CLUSTER_NAME, @@ -97,22 +94,21 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", + "--cluster-cidr=10.2.0.0/16", params.EnvParams["component_loglevel"], - }, + }...), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), }, pkiVolume(params)), "kube-scheduler": componentPod(api.Container{ Name: "kube-scheduler", - Image: params.EnvParams["hyperkube_image"], - Command: []string{ - "/hyperkube", - "scheduler", + Image: images.GetCoreImage(images.KubeSchedulerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), + Command: append(getImageEntrypoint("scheduler", params.Discovery.UseHyperkubeImage), []string{ "--leader-elect", MASTER, params.EnvParams["component_loglevel"], - }, + }...), LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), }), @@ -192,3 +188,11 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { }, } } + +func getImageEntrypoint(component string, useHyperkube bool) []string { + if useHyperkube { + return []string{"/hyperkube", component} + } + + return []string{"/usr/local/bin/kube-" + component} +} diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index e908c65f6c6..bbff5ad9190 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "crypto/rsa" @@ -137,8 +137,8 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 altNames certutil.AltNames // TODO actual SANs ) - if params.Discovery.ListenIP != "" { - altNames.IPs = append(altNames.IPs, net.ParseIP(params.Discovery.ListenIP)) + if params.Discovery.ListenIP != nil { + altNames.IPs = append(altNames.IPs, params.Discovery.ListenIP) } if params.Discovery.ApiServerDNSName != "" { diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index ff052575a77..844e5ffaafb 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubemaster +package master import ( "bytes" diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index e1da517d59d..0580bc37917 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubenode +package node import ( "fmt" diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index 3b41cdf0b27..62e43b8c4d3 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubenode +package node import ( "bytes" @@ -25,7 +25,7 @@ import ( "net/url" "strings" - "github.com/square/go-jose" + jose "github.com/square/go-jose" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" ) diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 041021bf9cb..dd6cd328358 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadmutil +package util import ( "fmt" diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 5d643dcf6e1..658044a306b 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadmutil +package util import ( "crypto/rand" From b9fd31ff7e69321841870c5d3afbfc75b7dc7bd0 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 7 Sep 2016 17:37:02 +0100 Subject: [PATCH 07/32] Refactoring improtant parts and start on docs --- cmd/kubeadm/app/kubeadm.go | 1 - pkg/kubeadm/README.md | 76 ++++++++++++++++++ pkg/kubeadm/api/types.go | 77 ++++++++++++++---- pkg/kubeadm/cmd/cmd.go | 22 ++--- pkg/kubeadm/cmd/init.go | 86 ++++++++++++++------ pkg/kubeadm/cmd/join.go | 39 +++++---- pkg/kubeadm/cmd/manual.go | 113 +++++++++++++++++--------- pkg/kubeadm/cmd/user.go | 2 +- pkg/kubeadm/images/images.go | 41 ++++------ pkg/kubeadm/master/addons.go | 69 ++++++++++------ pkg/kubeadm/master/discovery.go | 26 +++--- pkg/kubeadm/master/kubeconfig.go | 7 +- pkg/kubeadm/master/manifests.go | 133 ++++++++++++++++++------------- pkg/kubeadm/master/pki.go | 41 ++++++---- pkg/kubeadm/master/tokens.go | 20 ++--- pkg/kubeadm/node/csr.go | 10 +-- pkg/kubeadm/node/discovery.go | 18 ++--- pkg/kubeadm/util/kubeconfig.go | 8 +- pkg/kubeadm/util/tokens.go | 22 ++--- 19 files changed, 529 insertions(+), 282 deletions(-) create mode 100644 pkg/kubeadm/README.md diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 749948d5c34..3869a7057c1 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -46,7 +46,6 @@ func getEnvParams() map[string]string { "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), "etcd_image": "", "component_loglevel": "--v=4", - "dns_domain": "cluster.local", } for k := range envParams { diff --git a/pkg/kubeadm/README.md b/pkg/kubeadm/README.md new file mode 100644 index 00000000000..a43cde6a899 --- /dev/null +++ b/pkg/kubeadm/README.md @@ -0,0 +1,76 @@ +# Kubernetes Cluster Boostrap Made Easy + +## Usage + +### `kubeadm init` + +It's usually enough to run `kubeadm init`, but in some case you might like to override the +default behaviour. + +- `--token=` + +By default, a token is generated, but if you are to automate cluster deployment, you want to +set the token ahead of time. Read the docs for more information on the token format. + +- `--api-advertise-addr=` (multiple values allowed) +- `--api-external-dns-name=` (multiple values allowed) + +By default, `kubeadm` will auto detect IP address and use that to generate API server certificates. +If you would like to access the API via any external IPs and/or DNS, which it might not be able +to detect, you can use `--api-advertise-addr` and `--api-external-dns-name` to add multiple +different IP addresses and DNS names. + +- `--service-cidr=` (default: "100.64/12") +- `--service-dns-domain=` (default: "cluster.local") + +- `--use-hyperkube=` (default: "false") + +***TODO(phase1+)*** + +- `--api-bind-addr=` +- `--api-bind-port=` + +***TODO(phase2)*** + +- `--api-bind-loopback-unsecure=` + +***TODO(pahse2)*** + +- `--prefer-private-network=` +- `--prefer-public-network=` + +### `kubeadm join` + +# User Experience Considerations + +> ***TODO*** _Move this into the design document + +a) `kube-apiserver` will listen on `0.0.0.0:443` and `127.0.0.1:8080`, which is not configurable right now and make things a bit easier for the MVP +b) there is `kube-discovery`, which will listen on `0.0.0.0:9898` + + +from the point of view of `kubeadm init`, we need to have +a) a primary IP address as will be seen by the nodes and needs to go into the cert and `kube-discovery` configuration secret +b) some other names and addresses API server may be known by (e.g. external DNS and/or LB and/or NAT) + +from that perspective we don’t can assume default ports for now, but for all the address we really only care about two ports (i.e. 443 and 9898) + +we should make ports configurable and expose some way of making API server bind to a specific address/interface + +but I think for the MVP we need solve the issue with hardcode IPs and DNS names in the certs + +so it sounds rather simple enough to introduce `--api-advertise-addr=` and `--api-external-dns-name=`, and allowing multiple of those sounds also simple enough + +from the `kubeadm join` perspective, it cares about the two ports we mentioned, and we can make those configurable too + +but for now it’s easier to pass just the IP address + +plust it’s also require, so passing it without a named flag sounds convenient, and it’s something users are familiar with + +that’s what Consul does, what what Weave does, and now Docker SwarmKit does the same thing also (edited) + +flags will differ, as there are some Kubernetes-specifics aspects to what join does, but basic join semantics will remain _familiar_ + +if we do marry `kube-discovery` with the API, we might do `kubeadm join host:port`, as we’d end-up with a single port to care about + +but we haven’t yet diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go index dab8765e871..b71957172f9 100644 --- a/pkg/kubeadm/api/types.go +++ b/pkg/kubeadm/api/types.go @@ -20,29 +20,76 @@ import ( "net" ) -type BootstrapParams struct { - // TODO this is mostly out of date and bloated now, let's revisit this soon - Discovery *OutOfBandDiscovery - EnvParams map[string]string +type KubeadmConfig struct { + InitFlags + JoinFlags + ManualFlags + Secrets struct { + GivenToken string // dot-separated `.` set by the user + TokenID string // optional on master side, will be generated if not specified + Token []byte // optional on master side, will be generated if not specified + BearerToken string // set based on Token + } + EnvParams map[string]string // TODO(phase2) this is likely to be come componentconfig } -type OutOfBandDiscovery struct { - // 'join-node' side +// TODO(phase2) should we add validatin funcs on these structs? + +type InitFlags struct { + API struct { + AdvertiseAddrs []net.IP + ExternalDNSName []string + } + Services struct { + CIDR net.IPNet + DNSDomain string + } + CloudProvider string +} + +const ( + DefaultServiceDNSDomain = "cluster.local" + DefaultServicesCIDRString = "100.64.0.0/12" +) + +var ( + DefaultServicesCIDR *net.IPNet + ListOfCloudProviders = []string{ + "aws", + "azure", + "cloudstack", + "gce", + "mesos", + "openstack", + "ovirt", + "rackspace", + "vsphere", + } + SupportedCloudProviders map[string]bool +) + +func init() { + _, DefaultServicesCIDR, _ = net.ParseCIDR(DefaultServicesCIDRString) + SupportedCloudProviders = make(map[string]bool, len(ListOfCloudProviders)) + for _, v := range ListOfCloudProviders { + SupportedCloudProviders[v] = true + } +} + +type JoinFlags struct { + MasterAddrs []net.IP +} + +// TODO(phase1?) we haven't decided whether manual sub commands should get merged into main commands... +type ManualFlags struct { ApiServerURLs string // comma separated CaCertFile string - GivenToken string // dot-separated `.` set by the user - TokenID string // optional on master side, will be generated if not specified - Token []byte // optional on master side, will be generated if not specified BearerToken string // set based on Token - // 'init-master' side - ApiServerDNSName string // optional, used in master bootstrap - ListenIP net.IP // optional IP for master to listen on, rather than autodetect - UseHyperkubeImage bool + ListenIP net.IP // optional IP for master to listen on, rather than autodetect } type ClusterInfo struct { - // TODO Kind, apiVersion - // TODO clusterId, fetchedTime, expiredTime + // TODO(pahse1?) this may become simply `api.Config` CertificateAuthorities []string `json:"certificateAuthorities"` Endpoints []string `json:"endpoints"` } diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index c152f768ec6..dda31030e9a 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -73,25 +73,25 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env // TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly // i.e. make it automation friendly // - // TODO(phase2) create an bastraction that defines files and the content that needs to + // TODO(phase2) create an abstraction that defines files and the content that needs to // be written to disc and write it all in one go at the end as we have a lot of // crapy little files written from different parts of this code; this could also // be useful for testing - bootstrapParams := &kubeadmapi.BootstrapParams{ - Discovery: &kubeadmapi.OutOfBandDiscovery{ - // TODO(phase1) this type no longer makes sense here - }, - EnvParams: envParams, - } + s := new(kubeadmapi.KubeadmConfig) + s.EnvParams = envParams + + //s.InitFlags, s.JoinFlags = new(kubeadmapi.InitFlags), new(kubeadmapi.JoinFlags) + + //s.ManualFlags = new(kubeadmapi.ManualFlags) cmds.ResetFlags() cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) - cmds.AddCommand(NewCmdInit(out, bootstrapParams)) - cmds.AddCommand(NewCmdJoin(out, bootstrapParams)) - cmds.AddCommand(NewCmdUser(out, bootstrapParams)) - cmds.AddCommand(NewCmdManual(out, bootstrapParams)) + cmds.AddCommand(NewCmdInit(out, s)) + cmds.AddCommand(NewCmdJoin(out, s)) + cmds.AddCommand(NewCmdUser(out, s)) + cmds.AddCommand(NewCmdManual(out, s)) return cmds } diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index 56422cf6439..73baa5cc465 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -40,52 +41,91 @@ var ( `) ) -func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { + advertiseAddrs := &[]string{} cmd := &cobra.Command{ Use: "init", Short: "Run this on the first server you deploy onto.", Run: func(cmd *cobra.Command, args []string) { - err := RunInit(out, cmd, args, params) - cmdutil.CheckErr(err) + err := RunInit(out, cmd, args, s, advertiseAddrs) + cmdutil.CheckErr(err) // TODO(phase1+) append alpha warning with bugs URL etc }, } - cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, - `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVar(¶ms.Discovery.GivenToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) - cmd.PersistentFlags().BoolVar(¶ms.Discovery.UseHyperkubeImage, "use-hyperkube", false, - `(optional) Use the hyperkube image for running the apiserver, controller-manager, scheduler and proxy.`) + cmd.PersistentFlags().StringVar( + &s.Secrets.GivenToken, "token", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, + ) + cmd.PersistentFlags().StringSliceVar( + advertiseAddrs, "api-advertise-addr", []string{}, + `(optional) IP address to advertise, in case autodetection fails.`, + ) + cmd.PersistentFlags().StringSliceVar( + &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, + `(optional) DNS name to advertise, in case you have configured one yourself.`, + ) + + cmd.PersistentFlags().IPNetVar( + &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, + `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, + `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.CloudProvider, "cloud-provider", "", + `(optional) enable cloud proiver features (external load-balancers, storage, etc)`, + ) return cmd } -func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - +func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { // Auto-detect the IP - if params.Discovery.ListenIP == nil { + if len(*advertiseAddrs) == 0 { + // TODO(phase1+) perhaps we could actually grab eth0 and eth1 ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip + s.InitFlags.API.AdvertiseAddrs = []net.IP{ip} + } else { + for _, i := range *advertiseAddrs { + addr := net.ParseIP(i) + if addr == nil { + return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) + } + s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) + } } - if err := kubemaster.CreateTokenAuthFile(params); err != nil { + + if s.InitFlags.CloudProvider != "" { + // TODO(phase2) we should be able to auto-detect it and check whether things like IAM roles are correct + if _, ok := kubeadmapi.SupportedCloudProviders[s.InitFlags.CloudProvider]; !ok { + return fmt.Errorf(" cloud provider %q is not supported, you can use any of %v, or leave it unset", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders) + } + } + + if err := kubemaster.CreateTokenAuthFile(s); err != nil { return err } - if err := kubemaster.WriteStaticPodManifests(params); err != nil { + + if err := kubemaster.WriteStaticPodManifests(s); err != nil { return err } - caKey, caCert, err := kubemaster.CreatePKIAssets(params) + + caKey, caCert, err := kubemaster.CreatePKIAssets(s) if err != nil { return err } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert) if err != nil { return err } for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { return err } } @@ -99,18 +139,18 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmap return err } - if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil { + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(s, client, caCert); err != nil { return err } - if err := kubemaster.CreateEssentialAddons(params, client); err != nil { + if err := kubemaster.CreateEssentialAddons(s, client); err != nil { return err } - // TODO use templates to reference struct fields directly as order of args is fragile + // TODO(phase1+) use templates to reference struct fields directly as order of args is fragile fmt.Fprintf(out, init_done_msgf, - params.Discovery.GivenToken, - params.Discovery.ListenIP, + s.Secrets.GivenToken, + s.InitFlags.API.AdvertiseAddrs[0].String(), ) return nil diff --git a/pkg/kubeadm/cmd/join.go b/pkg/kubeadm/cmd/join.go index 5c8f7a1ffb3..c83f6663069 100644 --- a/pkg/kubeadm/cmd/join.go +++ b/pkg/kubeadm/cmd/join.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -40,44 +41,52 @@ var ( `) ) -func NewCmdJoin(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "join", Short: "Run this on other servers to join an existing cluster.", Run: func(cmd *cobra.Command, args []string) { - err := RunJoin(out, cmd, args, params) + err := RunJoin(out, cmd, args, s) cmdutil.CheckErr(err) }, } // TODO this should become `kubeadm join --token=<...> ` - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", - `Comma separated list of API server URLs. Typically this might be just - https://:8080/`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", - `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + cmd.PersistentFlags().StringVarP( + &s.Secrets.GivenToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`, + ) return cmd } -func RunJoin(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - ok, err := kubeadmutil.UseGivenTokenIfValid(params) +func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { + // TODO this we are missing args from the help text, there should be a way to tell cobra about it + if len(args) == 0 { + return fmt.Errorf(" must specify master IP address (see --help)") + } + for _, i := range args { + addr := net.ParseIP(i) // TODO(phase1+) should allow resolvable names too + if addr == nil { + return fmt.Errorf(" failed parse argument (%q) as an IP address", i) + } + s.JoinFlags.MasterAddrs = append(s.JoinFlags.MasterAddrs, addr) + } + + ok, err := kubeadmutil.UseGivenTokenIfValid(s) if !ok { if err != nil { - return fmt.Errorf("%s (see --help)\n", err) + return fmt.Errorf(" %s (see --help)\n", err) } return fmt.Errorf("Must specify --token (see --help)\n") } - if params.Discovery.ApiServerURLs == "" { - return fmt.Errorf("Must specify --api-server-urls (see --help)\n") - } - kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(params) + kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(s) if err != nil { return err } - err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig) if err != nil { return err } diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index 49a611ced12..a6d0e40d894 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -58,23 +59,23 @@ var ( `) ) -// TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken +// TODO --token here becomes `s.Secrets.BearerToken` and not `s.Secrets.GivenToken` // may be we should make it the same and ask user to pass dot-separated tokens // in any of the modes; we could also enable discovery API in the manual mode just // as well, there is no reason we shouldn't let user mix and match modes, unless // it is too difficult to support -func NewCmdManual(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManual(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "manual", Short: "Advanced, less-automated functionality, for power users.", // TODO put example usage in the Long description here } - cmd.AddCommand(NewCmdManualBootstrap(out, params)) + cmd.AddCommand(NewCmdManualBootstrap(out, s)) return cmd } -func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrap(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "bootstrap", Short: "Manually bootstrap a cluster 'out-of-band'", @@ -85,13 +86,14 @@ func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *c Run: func(cmd *cobra.Command, args []string) { }, } - cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, params)) - cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, params)) + cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, s)) + cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, s)) return cmd } -func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { + advertiseAddrs := &[]string{} cmd := &cobra.Command{ Use: "init-master", Short: "Manually bootstrap a master 'out-of-band'", @@ -101,101 +103,140 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap components. `), Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapInitMaster(out, cmd, args, params) + err := RunManualBootstrapInitMaster(out, cmd, args, s, advertiseAddrs) cmdutil.CheckErr(err) }, } - params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server - cmd.PersistentFlags().StringVar(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", - `(optional) DNS name for the API server, will be encoded into - subjectAltName in the resulting (generated) TLS certificates`) - cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, - `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVar(¶ms.Discovery.BearerToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + cmd.PersistentFlags().StringVar( + &s.Secrets.BearerToken, "token", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, + ) + cmd.PersistentFlags().StringSliceVar( + advertiseAddrs, "api-advertise-addr", nil, + `(optional) IP address to advertise, in case autodetection fails.`, + ) + cmd.PersistentFlags().StringSliceVar( + &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, + `(optional) DNS name to advertise, in case you have configured one yourself.`, + ) + _, defaultServicesCIDR, _ := net.ParseCIDR("100.64.0.0/12") + cmd.PersistentFlags().IPNetVar( + &s.InitFlags.Services.CIDR, "service-cidr", *defaultServicesCIDR, + `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.Services.DNSDomain, "service-dns-domain", "cluster.local", + `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, + ) return cmd } -func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { +func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { // Auto-detect the IP - if params.Discovery.ListenIP == nil { + if len(*advertiseAddrs) == 0 { + // TODO(phase1+) perhaps we could actually grab eth0 and eth1 ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip + s.InitFlags.API.AdvertiseAddrs = []net.IP{ip} + } else { + for _, i := range *advertiseAddrs { + addr := net.ParseIP(i) + if addr == nil { + return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) + } + s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) + } } - if err := kubemaster.CreateTokenAuthFile(params); err != nil { + if err := kubemaster.CreateTokenAuthFile(s); err != nil { return err } - if err := kubemaster.WriteStaticPodManifests(params); err != nil { + if err := kubemaster.WriteStaticPodManifests(s); err != nil { return err } - caKey, caCert, err := kubemaster.CreatePKIAssets(params) + caKey, caCert, err := kubemaster.CreatePKIAssets(s) if err != nil { return err } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert) if err != nil { return err } for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { return err } } + // TODO we have most of cmd/init functionality here, except for `CreateDiscoveryDeploymentAndSecret()` + // it may be a good idea to just merge the two commands into one, and it's something we have started talking + // about, the only question is where disco service should be an opt-out... + + client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"]) + if err != nil { + return err + } + + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil { + return err + } + + if err := kubemaster.CreateEssentialAddons(s, client); err != nil { + return err + } + // TODO use templates to reference struct fields directly as order of args is fragile fmt.Fprintf(out, manual_init_done_msgf, - params.Discovery.BearerToken, - params.Discovery.ListenIP, + s.Secrets.BearerToken, + s.InitFlags.API.AdvertiseAddrs[0].String(), ) return nil } -func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrapJoinNode(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "join-node", Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapJoinNode(out, cmd, args, params) + err := RunManualBootstrapJoinNode(out, cmd, args, s) cmdutil.CheckErr(err) }, } - cmd.PersistentFlags().StringVarP(¶ms.Discovery.CaCertFile, "ca-cert-file", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.CaCertFile, "ca-cert-file", "", "", `Path to a CA cert file in PEM format. The same CA cert must be distributed to all servers.`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.ApiServerURLs, "api-server-urls", "", "", `Comma separated list of API server URLs. Typically this might be just https://:8080/`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.BearerToken, "token", "", "", `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) return cmd } -func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - if params.Discovery.CaCertFile == "" { +func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { + if s.ManualFlags.CaCertFile == "" { fmt.Fprintf(out, "Must specify --ca-cert-file (see --help)\n") return nil } - if params.Discovery.ApiServerURLs == "" { + if s.ManualFlags.ApiServerURLs == "" { fmt.Fprintf(out, "Must specify --api-server-urls (see --help)\n") return nil } - kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) + kubeconfig, err := kubenode.PerformTLSBootstrapFromConfig(s) if err != nil { fmt.Fprintf(out, "Failed to perform TLS bootstrap: %s\n", err) return err } - err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig) if err != nil { fmt.Fprintf(out, "Unable to write config for node:\n%s\n", err) return err diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go index 3d30f50d6e5..6beda1bf40e 100644 --- a/pkg/kubeadm/cmd/user.go +++ b/pkg/kubeadm/cmd/user.go @@ -23,7 +23,7 @@ import ( kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" ) -func NewCmdUser(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdUser(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "user", Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go index 27f3fb03a60..6452456c52b 100644 --- a/pkg/kubeadm/images/images.go +++ b/pkg/kubeadm/images/images.go @@ -24,53 +24,42 @@ import ( const ( KubeEtcdImage = "etcd" - KubeApiServerImage = "apiserver" + KubeApiServerImage = "apiserver" KubeControllerManagerImage = "controller-manager" - KubeSchedulerImage = "scheduler" - KubeProxyImage = "proxy" + KubeSchedulerImage = "scheduler" + KubeProxyImage = "proxy" - KubeDnsImage = "kube-dns" - KubeDnsmasqImage = "dnsmasq" + KubeDnsImage = "kube-dns" + KubeDnsmasqImage = "dnsmasq" KubeExechealthzImage = "exechealthz" - - gcrPrefix = "gcr.io/google_containers" + gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" kubeVersion = "v1.4.0-alpha.3" - kubeDnsVersion = "1.7" - dnsmasqVersion = "1.3" + kubeDnsVersion = "1.7" + dnsmasqVersion = "1.3" exechealthzVersion = "1.1" ) -func GetCoreImage(image string, overrideImage string, useHyperkube bool) string { +func GetCoreImage(image string, overrideImage string) string { if overrideImage != "" { return overrideImage } - if useHyperkube { - return map[string]string{ - KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - }[image] - } - return map[string]string{ - KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), + KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), }[image] } func GetAddonImage(image string) string { return map[string]string{ - KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dns", runtime.GOARCH, kubeDnsVersion), - KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), + KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDnsVersion), + KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), }[image] } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index add2674d3fd..362d6112411 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -25,20 +25,18 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" "k8s.io/kubernetes/pkg/kubeadm/images" + ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) -func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { privilegedTrue := true return api.PodSpec{ SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ - Name: "kube-proxy", - Image: images.GetCoreImage(images.KubeProxyImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("proxy", params.Discovery.UseHyperkubeImage), []string{ - "--kubeconfig=/run/kubeconfig", - params.EnvParams["component_loglevel"], - }...), + Name: kubeProxy, + Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]), + Command: append(getComponentCommand("proxy", s), "--kubeconfig=/run/kubeconfig"), SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, VolumeMounts: []api.VolumeMount{ { @@ -67,7 +65,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { { Name: "kubeconfig", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: path.Join(params.EnvParams["prefix"], "kubelet.conf")}, + HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["prefix"], "kubelet.conf")}, }, }, { @@ -80,8 +78,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -// TODO(phase1): Create the DNS service as well -func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { dnsPodResources := api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("100m"), @@ -93,6 +90,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { api.ResourceName(api.ResourceMemory): resource.MustParse("50Mi"), } + kubeDNSPort := int32(10053) + dnsmasqPort := int32(53) + + nslookup := fmt.Sprintf("nslookup kubernetes.default.svc.%s 127.0.0.1", s.InitFlags.Services.DNSDomain) + + nslookup = fmt.Sprintf("-cmd=%s:%d >/dev/null && %s:%d >/dev/null", + nslookup, dnsmasqPort, + nslookup, kubeDNSPort, + ) + return api.PodSpec{ Containers: []api.Container{ // DNS server @@ -104,8 +111,8 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Requests: dnsPodResources, }, Args: []string{ - "--domain=" + params.EnvParams["dns_domain"], - "--dns-port=10053", + fmt.Sprintf("--domain=%s", s.InitFlags.Services.DNSDomain), + fmt.Sprintf("--dns-port=%d", kubeDNSPort), // TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__ }, LivenessProbe: &api.Probe{ @@ -136,12 +143,12 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { }, Ports: []api.ContainerPort{ { - ContainerPort: 10053, + ContainerPort: kubeDNSPort, Name: "dns-local", Protocol: api.ProtocolUDP, }, { - ContainerPort: 10053, + ContainerPort: kubeDNSPort, Name: "dns-tcp-local", Protocol: api.ProtocolTCP, }, @@ -158,16 +165,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Args: []string{ "--cache-size=1000", "--no-resolv", - "--server=127.0.0.1#10053", + fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort), }, Ports: []api.ContainerPort{ { - ContainerPort: 53, + ContainerPort: dnsmasqPort, Name: "dns", Protocol: api.ProtocolUDP, }, { - ContainerPort: 53, + ContainerPort: dnsmasqPort, Name: "dns-tcp", Protocol: api.ProtocolTCP, }, @@ -182,7 +189,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Requests: healthzPodResources, }, Args: []string{ - "-cmd=nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1:10053 >/dev/null", + nslookup, "-port=8080", "-quiet", }, @@ -196,19 +203,27 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func createKubeDNSServiceSpec(params *kubeadmapi.BootstrapParams) api.ServiceSpec { - return api.ServiceSpec{ + +func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, error) { + ip, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 10) + if err != nil { + return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%s]", s.InitFlags.Services.CIDR, err) + } + + svc := &api.ServiceSpec{ Selector: map[string]string{"name": "kube-dns"}, Ports: []api.ServicePort{ {Name: "dns", Port: 53, Protocol: api.ProtocolUDP}, {Name: "dns-tcp", Port: 53, Protocol: api.ProtocolTCP}, }, - ClusterIP: "100.64.0.2", + ClusterIP: ip.String(), } + + return svc, nil } -func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { - kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) +func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset) error { + kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(s)) SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { @@ -217,15 +232,19 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset fmt.Println(" created essential addon: kube-proxy") - kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(params)) + kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(s)) SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } - kubeDNSService := NewService("kube-dns", createKubeDNSServiceSpec(params)) + kubeDNSServiceSpec, err := createKubeDNSServiceSpec(s) + if err != nil { + return fmt.Errorf(" failed creating essential kube-dns addon - %s", err) + } + kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec) if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go index 15ae71e9969..4ae8dcf13e6 100644 --- a/pkg/kubeadm/master/discovery.go +++ b/pkg/kubeadm/master/discovery.go @@ -39,7 +39,7 @@ const ( kubeDiscoverySecretName = "clusterinfo" ) -func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) map[string][]byte { +func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte { // TODO ListenIP is probably not the right now, although it's best we have right now // if user provides a DNS name, or anything else, we should use that, may be it's really // the list of all SANs (minus internal DNS names and service IP)? @@ -50,8 +50,11 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x tokenMap = map[string]string{} ) - endpointList = append(endpointList, fmt.Sprintf("https://%s:443", params.Discovery.ListenIP)) - tokenMap[params.Discovery.TokenID] = hex.EncodeToString(params.Discovery.Token) + for _, addr := range s.InitFlags.API.AdvertiseAddrs { + endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String())) + } + + tokenMap[s.Secrets.TokenID] = hex.EncodeToString(s.Secrets.Token) data["endpoint-list.json"], _ = json.Marshal(endpointList) data["token-map.json"], _ = json.Marshal(tokenMap) @@ -60,7 +63,7 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x return data } -func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { return api.PodSpec{ // We have to use host network namespace, as `HostPort`/`HostIP` are Docker's // buisness and CNI support isn't quite there yet (except for kubenet) @@ -69,7 +72,7 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ Name: kubeDiscoverynName, - Image: params.EnvParams["discovery_image"], + Image: s.EnvParams["discovery_image"], Command: []string{"/usr/bin/kube-discovery"}, VolumeMounts: []api.VolumeMount{{ Name: kubeDiscoverySecretName, @@ -77,7 +80,8 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { ReadOnly: true, }}, Ports: []api.ContainerPort{ - // TODO when CNI issue (#31307) is resolved, we should add `HostIP: params.Discovery.ListenIP` + // TODO when CNI issue (#31307) is resolved, we should consider adding + // `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address` {Name: "http", ContainerPort: 9898, HostPort: 9898}, }, }}, @@ -90,13 +94,13 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery { +func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery { kd := kubeDiscovery{ - Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)), + Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(s)), Secret: &api.Secret{ ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, Type: api.SecretTypeOpaque, - Data: encodeKubeDiscoverySecretData(params, caCert), + Data: encodeKubeDiscoverySecretData(s, caCert), }, } @@ -106,8 +110,8 @@ func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certifica return kd } -func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error { - kd := newKubeDiscovery(params, caCert) +func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset, caCert *x509.Certificate) error { + kd := newKubeDiscovery(s, caCert) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoverynName, err) diff --git a/pkg/kubeadm/master/kubeconfig.go b/pkg/kubeadm/master/kubeconfig.go index ea76c3849b6..3287ea7a703 100644 --- a/pkg/kubeadm/master/kubeconfig.go +++ b/pkg/kubeadm/master/kubeconfig.go @@ -28,11 +28,14 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -func CreateCertsAndConfigForClients(params *kubeadmapi.BootstrapParams, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { +func CreateCertsAndConfigForClients(s *kubeadmapi.KubeadmConfig, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { basicClientConfig := kubeadmutil.CreateBasicClientConfig( "kubernetes", - fmt.Sprintf("https://%s:443", params.Discovery.ListenIP), + // TODO this is not great, but there is only one address we can use here + // so we'll pick the first one, there is much of chance to have an empty + // slice by the time this gets called + fmt.Sprintf("https://%s:443", s.InitFlags.API.AdvertiseAddrs[0]), certutil.EncodeCertPEM(caCert), ) diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 385813dffe9..f2a985b14bf 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -36,85 +36,65 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=100.64.0.0/12" - CLUSTER_NAME = "--cluster-name=kubernetes" - MASTER = "--master=127.0.0.1:8080" + DefaultClusterName = "--cluster-name=kubernetes" + + etcd = "etcd" + apiServer = "apiserver" + controllerManager = "controller-manager" + scheduler = "scheduler" + proxy = "proxy" + kubeAPIServer = "kube-apiserver" + kubeControllerManager = "kube-controller-manager" + kubeScheduler = "kube-scheduler" + kubeProxy = "kube-proxy" ) // TODO look into what this really means, scheduler prints it for some reason // //E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' -func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { +func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume - "etcd": componentPod(api.Container{ + etcd: componentPod(api.Container{ Command: []string{ "/usr/local/bin/etcd", "--listen-client-urls=http://127.0.0.1:2379", "--advertise-client-urls=http://127.0.0.1:2379", "--data-dir=/var/etcd/data", }, - Image: images.GetCoreImage(images.KubeEtcdImage, params.EnvParams["etcd_image"], params.Discovery.UseHyperkubeImage), + Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), - Name: "etcd-server", + Name: etcd, Resources: componentResources("200m"), }), // TODO bind-mount certs in - "kube-apiserver": componentPod(api.Container{ - Name: "kube-apiserver", - Image: images.GetCoreImage(images.KubeApiServerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("apiserver", params.Discovery.UseHyperkubeImage), []string{ - "--address=127.0.0.1", - "--etcd-servers=http://127.0.0.1:2379", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", - SERVICE_CLUSTER_IP_RANGE, - "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--client-ca-file=/etc/kubernetes/pki/ca.pem", - "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", - "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--secure-port=443", - "--allow-privileged", - params.EnvParams["component_loglevel"], - "--token-auth-file=/etc/kubernetes/pki/tokens.csv", - }...), + kubeAPIServer: componentPod(api.Container{ + Name: kubeAPIServer, + Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(apiServer, s), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), - }, pkiVolume(params)), - "kube-controller-manager": componentPod(api.Container{ - Name: "kube-controller-manager", - Image: images.GetCoreImage(images.KubeControllerManagerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("controller-manager", params.Discovery.UseHyperkubeImage), []string{ - "--leader-elect", - MASTER, - CLUSTER_NAME, - "--root-ca-file=/etc/kubernetes/pki/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", - "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", - "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", - "--cluster-cidr=10.2.0.0/16", - params.EnvParams["component_loglevel"], - }...), + }, pkiVolume(s)), + kubeControllerManager: componentPod(api.Container{ + Name: kubeControllerManager, + Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(controllerManager, s), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), - }, pkiVolume(params)), - "kube-scheduler": componentPod(api.Container{ - Name: "kube-scheduler", - Image: images.GetCoreImage(images.KubeSchedulerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("scheduler", params.Discovery.UseHyperkubeImage), []string{ - "--leader-elect", - MASTER, - params.EnvParams["component_loglevel"], - }...), + }, pkiVolume(s)), + kubeScheduler: componentPod(api.Container{ + Name: kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(scheduler, s), LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), }), } - manifestsPath := path.Join(params.EnvParams["prefix"], "manifests") + manifestsPath := path.Join(s.EnvParams["prefix"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) } @@ -131,11 +111,11 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { return nil } -func pkiVolume(params *kubeadmapi.BootstrapParams) api.Volume { +func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: params.EnvParams["host_pki_path"]}, + HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_pki_path"]}, }, } } @@ -189,10 +169,51 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { } } -func getImageEntrypoint(component string, useHyperkube bool) []string { - if useHyperkube { - return []string{"/hyperkube", component} +func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { + baseFalgs := map[string][]string{ + apiServer: []string{ + "--address=127.0.0.1", + "--etcd-servers=http://127.0.0.1:2379", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", + "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), + "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--client-ca-file=/etc/kubernetes/pki/ca.pem", + "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", + "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--secure-port=443", + "--allow-privileged", + "--token-auth-file=/etc/kubernetes/pki/tokens.csv", + }, + controllerManager: []string{ + "--leader-elect", + "--master=127.0.0.1:8080", + DefaultClusterName, + "--root-ca-file=/etc/kubernetes/pki/ca.pem", + "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", + "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", + "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), + }, + scheduler: []string{ + "--leader-elect", + "--master=127.0.0.1:8080", + }, + proxy: []string{}, } - return []string{"/usr/local/bin/kube-" + component} + if s.EnvParams["hyperkube_image"] != "" { + command = []string{"/hyperkube", component} + } else { + command = []string{"/usr/local/bin/kube-" + component} + } + + command = append(command, s.EnvParams["component_loglevel"]) + command = append(command, baseFalgs[component]...) + + if component == controllerManager && s.InitFlags.CloudProvider != "" { + command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) + } + + return } diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index bbff5ad9190..19c8ccac927 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -20,10 +20,10 @@ import ( "crypto/rsa" "crypto/x509" "fmt" - "net" "path" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" certutil "k8s.io/kubernetes/pkg/util/cert" ) @@ -51,19 +51,26 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { return key, cert, nil } -func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { +func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) } - // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly - altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"), net.ParseIP("10.16.0.1"), net.ParseIP("100.64.0.1")) - altNames.DNSNames = append(altNames.DNSNames, + + internalAPIServerFQDN := []string{ "kubernetes", "kubernetes.default", "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - ) + fmt.Sprintf("kubernetes.default.svc.%s", s.InitFlags.Services.DNSDomain), + } + + internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 1) + if err != nil { + return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]") + } + + altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP) + altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...) config := certutil.CertConfig{ CommonName: "kube-apiserver", @@ -131,21 +138,21 @@ func newServiceAccountKey() (*rsa.PrivateKey, error) { return key, nil } -func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509.Certificate, error) { +func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certificate, error) { var ( err error - altNames certutil.AltNames // TODO actual SANs + altNames certutil.AltNames ) - if params.Discovery.ListenIP != nil { - altNames.IPs = append(altNames.IPs, params.Discovery.ListenIP) + if len(s.InitFlags.API.AdvertiseAddrs) > 0 { + altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...) } - if params.Discovery.ApiServerDNSName != "" { - altNames.DNSNames = append(altNames.DNSNames, params.Discovery.ApiServerDNSName) + if len(s.InitFlags.API.ExternalDNSName) > 0 { + altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSName...) } - pkiPath := path.Join(params.EnvParams["host_pki_path"]) + pkiPath := path.Join(s.EnvParams["host_pki_path"]) caKey, caCert, err := newCertificateAuthority() if err != nil { @@ -156,7 +163,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %s", err) } - apiKey, apiCert, err := newServerKeyAndCert(caCert, caKey, altNames) + apiKey, apiCert, err := newServerKeyAndCert(s, caCert, caKey, altNames) if err != nil { return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %s", err) } @@ -174,7 +181,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) } - // TODO print a summary of SANs used and checksums (signatures) of each of the certiicates - fmt.Printf(" created keys and certificates in %q\n", params.EnvParams["host_pki_path"]) + // TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certiicates + fmt.Printf(" created keys and certificates in %q\n", pkiPath) return caKey, caCert, nil } diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index 844e5ffaafb..f741105d588 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -28,17 +28,17 @@ import ( "k8s.io/kubernetes/pkg/util/uuid" ) -func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { - ok, err := kubeadmutil.UseGivenTokenIfValid(params) +func generateTokenIfNeeded(s *kubeadmapi.KubeadmConfig) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(s) if !ok { if err != nil { return err } - err = kubeadmutil.GenerateToken(params) + err = kubeadmutil.GenerateToken(s) if err != nil { return err } - fmt.Printf(" generated token: %q\n", params.Discovery.GivenToken) + fmt.Printf(" generated token: %q\n", s.Secrets.GivenToken) } else { fmt.Println(" accepted provided token") } @@ -46,15 +46,15 @@ func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { return nil } -func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { - tokenAuthFilePath := path.Join(params.EnvParams["host_pki_path"], "tokens.csv") - if err := generateTokenIfNeeded(params); err != nil { +func CreateTokenAuthFile(s *kubeadmapi.KubeadmConfig) error { + tokenAuthFilePath := path.Join(s.EnvParams["host_pki_path"], "tokens.csv") + if err := generateTokenIfNeeded(s); err != nil { return fmt.Errorf(" failed to generate token(s) [%s]", err) } - if err := os.MkdirAll(params.EnvParams["host_pki_path"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) + if err := os.MkdirAll(s.EnvParams["host_pki_path"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["host_pki_path"], err) } - serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) + serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", s.Secrets.BearerToken, uuid.NewUUID())) if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil { return fmt.Errorf(" failed to save token auth file (%q) [%s]", tokenAuthFilePath, err) } diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index 0580bc37917..d4f4f0eef5d 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -34,17 +34,17 @@ func getNodeName() string { return "TODO" } -func PerformTLSBootstrapFromParams(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { - caCert, err := ioutil.ReadFile(params.Discovery.CaCertFile) +func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { + caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile) if err != nil { return nil, fmt.Errorf(" failed to load CA certificate [%s]", err) } - return PerformTLSBootstrap(params, strings.Split(params.Discovery.ApiServerURLs, ",")[0], caCert) + return PerformTLSBootstrap(s, strings.Split(s.ManualFlags.ApiServerURLs, ",")[0], caCert) } // Create a restful client for doing the certificate signing request. -func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { +func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { // TODO try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) @@ -52,7 +52,7 @@ func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( *kubeadmutil.MakeClientConfigWithToken( - bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), params.Discovery.BearerToken, + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), s.Secrets.BearerToken, ), &clientcmd.ConfigOverrides{}, ).ClientConfig() diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index 62e43b8c4d3..8ec04b10d17 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -22,23 +22,15 @@ import ( "fmt" "io" "net/http" - "net/url" - "strings" jose "github.com/square/go-jose" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" ) -func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { - firstURL := strings.Split(params.Discovery.ApiServerURLs, ",")[0] // TODO obviously we should do something better.. . - apiServerURL, err := url.Parse(firstURL) - if err != nil { - return nil, fmt.Errorf(" failed to parse given API server URL (%q) [%s]", firstURL, err) - } - - host, port := strings.Split(apiServerURL.Host, ":")[0], 9898 // TODO this is too naive - requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, params.Discovery.TokenID) +func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { + host, port := s.JoinFlags.MasterAddrs[0].String(), 9898 + requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, s.Secrets.TokenID) req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) @@ -61,7 +53,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda fmt.Println(" cluster info object received, verifying signature using given token") - output, err := object.Verify(params.Discovery.Token) + output, err := object.Verify(s.Secrets.Token) if err != nil { return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%s]", err) } @@ -84,5 +76,5 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda apiServer := clusterInfo.Endpoints[0] caCert := []byte(clusterInfo.CertificateAuthorities[0]) - return PerformTLSBootstrap(params, apiServer, caCert) + return PerformTLSBootstrap(s, apiServer, caCert) } diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index dd6cd328358..4fe56cf03f6 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -81,12 +81,12 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // running in that case - they'd need to stop the kubelet, remove the file, and // start it again in that case). -func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { - if err := os.MkdirAll(params.EnvParams["prefix"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["prefix"], err) +func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { + if err := os.MkdirAll(s.EnvParams["prefix"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["prefix"], err) } - filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + filename := path.Join(s.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile( filename, diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 658044a306b..da51ff551a4 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -42,7 +42,7 @@ func randBytes(length int) ([]byte, string, error) { return b, hex.EncodeToString(b), nil } -func GenerateToken(params *kubeadmapi.BootstrapParams) error { +func GenerateToken(s *kubeadmapi.KubeadmConfig) error { _, tokenID, err := randBytes(TokenIDLen / 2) if err != nil { return err @@ -53,19 +53,19 @@ func GenerateToken(params *kubeadmapi.BootstrapParams) error { return err } - params.Discovery.TokenID = tokenID - params.Discovery.BearerToken = token - params.Discovery.Token = tokenBytes - params.Discovery.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) + s.Secrets.TokenID = tokenID + s.Secrets.BearerToken = token + s.Secrets.Token = tokenBytes + s.Secrets.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) return nil } -func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { - if params.Discovery.GivenToken == "" { +func UseGivenTokenIfValid(s *kubeadmapi.KubeadmConfig) (bool, error) { + if s.Secrets.GivenToken == "" { return false, nil // not given } fmt.Println(" validating provided token") - givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".") + givenToken := strings.Split(strings.ToLower(s.Secrets.GivenToken), ".") // TODO print desired format // TODO could also print more specific messages in each case invalidErr := " provided token is invalid - %s" @@ -86,8 +86,8 @@ func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { "length of second part is incorrect [%d (given) != %d (expected)]", len(tokenBytes), TokenBytes)) } - params.Discovery.TokenID = givenToken[0] - params.Discovery.BearerToken = givenToken[1] - params.Discovery.Token = tokenBytes + s.Secrets.TokenID = givenToken[0] + s.Secrets.BearerToken = givenToken[1] + s.Secrets.Token = tokenBytes return true, nil // given and valid } From b6d0efb85436ee32f7c68cba8f0e5e25ae582b3d Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 9 Sep 2016 09:26:06 -0300 Subject: [PATCH 08/32] Use string bytes for token instead of decoding hex. --- pkg/kubeadm/util/tokens.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index da51ff551a4..202a37b2451 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -77,15 +77,7 @@ func UseGivenTokenIfValid(s *kubeadmapi.KubeadmConfig) (bool, error) { "length of first part is incorrect [%d (given) != %d (expected) ]", len(givenToken[0]), TokenIDLen)) } - tokenBytes, err := hex.DecodeString(givenToken[1]) - if err != nil { - return false, fmt.Errorf(invalidErr, err) - } - if len(tokenBytes) != TokenBytes { - return false, fmt.Errorf(invalidErr, fmt.Sprintf( - "length of second part is incorrect [%d (given) != %d (expected)]", - len(tokenBytes), TokenBytes)) - } + tokenBytes := []byte(givenToken[1]) s.Secrets.TokenID = givenToken[0] s.Secrets.BearerToken = givenToken[1] s.Secrets.Token = tokenBytes From ac849dab8ed860a9bf63c25f3b39838d38fec7eb Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Fri, 9 Sep 2016 08:41:19 +0000 Subject: [PATCH 09/32] Check that sever API provides certificates capabilities Print user friendly error with information about required version. --- pkg/kubeadm/node/csr.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index d4f4f0eef5d..e6d841dd600 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -21,7 +21,10 @@ import ( "io/ioutil" "strings" + "k8s.io/kubernetes/pkg/apis/certificates" unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/client/typed/discovery" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" @@ -60,6 +63,12 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) } + err = checkCertsAPI(bootstrapClientConfig) + + if err != nil { + return nil, fmt.Errorf(" API compatibility error [%s]", err) + } + client, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) if err != nil { return nil, fmt.Errorf(" failed to create API client [%s]", err) @@ -88,3 +97,35 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert return finalConfig, nil } + +func checkCertsAPI(config *restclient.Config) error { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + + if err != nil { + return fmt.Errorf("failed to create API discovery client [%s]", err) + } + + serverGroups, err := discoveryClient.ServerGroups() + + if err != nil { + return fmt.Errorf("failed to retrieve a list of supported API objects [%s]", err) + } + + for _, group := range serverGroups.Groups { + if group.Name == certificates.GroupName { + return nil + } + } + + version, err := discoveryClient.ServerVersion() + serverVersion := version.String() + + if err != nil { + serverVersion = "N/A" + } + + // Due to changes in API namespaces for certificates + // https://github.com/kubernetes/kubernetes/pull/31887/ + // it is compatible only with versions released after v1.4.0-beta.0 + return fmt.Errorf("installed Kubernetes API server version \"%s\" does not support certificates signing request use v1.4.0 or newer", serverVersion) +} From a2a807b50d0d6a41e6d0fa179d906a33336f9056 Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Sun, 11 Sep 2016 13:39:35 +0000 Subject: [PATCH 10/32] Mount etcd data directory to host --- cmd/kubeadm/app/kubeadm.go | 1 + pkg/kubeadm/master/manifests.go | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 3869a7057c1..06b0a786b31 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -42,6 +42,7 @@ func getEnvParams() map[string]string { envParams := map[string]string{ "prefix": globalPrefix, "host_pki_path": path.Join(globalPrefix, "pki"), + "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), "etcd_image": "", diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index f2a985b14bf..2766e90d389 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -63,11 +63,12 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { "--advertise-client-urls=http://127.0.0.1:2379", "--data-dir=/var/etcd/data", }, + VolumeMounts: []api.VolumeMount{etcdVolumeMount()}, Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), Name: etcd, Resources: componentResources("200m"), - }), + }, etcdVolume(s)), // TODO bind-mount certs in kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, @@ -111,6 +112,22 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { return nil } +func etcdVolume(s *kubeadmapi.KubeadmConfig) api.Volume { + return api.Volume{ + Name: "etcd", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_etcd_path"]}, + }, + } +} + +func etcdVolumeMount() api.VolumeMount { + return api.VolumeMount{ + Name: "etcd", + MountPath: "/var/etcd", + } +} + func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", From 798bbaa83f77d07537c2ca4d26b0718fb739c69b Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 15 Sep 2016 14:40:47 +0200 Subject: [PATCH 11/32] Bump component image version tag --- pkg/kubeadm/images/images.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go index 6452456c52b..e1f5f48c95a 100644 --- a/pkg/kubeadm/images/images.go +++ b/pkg/kubeadm/images/images.go @@ -35,7 +35,7 @@ const ( gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" - kubeVersion = "v1.4.0-alpha.3" + kubeVersion = "v1.4.0-beta.6" kubeDnsVersion = "1.7" dnsmasqVersion = "1.3" From 6a20487b3886b8a9ccbbc7d5ed0096a5f80612c3 Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Thu, 15 Sep 2016 16:12:33 +0000 Subject: [PATCH 12/32] Add a flag to allow to schedule workload to the master node --- pkg/kubeadm/api/types.go | 1 + pkg/kubeadm/cmd/init.go | 6 +++++- pkg/kubeadm/cmd/manual.go | 6 +++++- pkg/kubeadm/master/apiclient.go | 15 +++++++++------ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go index b71957172f9..c6229a78d9c 100644 --- a/pkg/kubeadm/api/types.go +++ b/pkg/kubeadm/api/types.go @@ -45,6 +45,7 @@ type InitFlags struct { DNSDomain string } CloudProvider string + Schedulable bool } const ( diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index 73baa5cc465..691ef9d3e32 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -77,6 +77,10 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { &s.InitFlags.CloudProvider, "cloud-provider", "", `(optional) enable cloud proiver features (external load-balancers, storage, etc)`, ) + cmd.PersistentFlags().BoolVar( + &s.InitFlags.Schedulable, "schedule-workload", false, + `(optional) allow to schedule workload to the node`, + ) return cmd } @@ -135,7 +139,7 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub return err } - if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil { + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, s.Schedulable); err != nil { return err } diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index a6d0e40d894..a95601931ca 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -129,6 +129,10 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) &s.InitFlags.Services.DNSDomain, "service-dns-domain", "cluster.local", `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, ) + cmd.PersistentFlags().BoolVar( + &s.InitFlags.Schedulable, "schedule-workload", false, + `(optional) allow to schedule workload to the node`, + ) return cmd } @@ -181,7 +185,7 @@ func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []stri return err } - if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil { + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, s.Schedulable); err != nil { return err } diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go index 97654cbab52..777ff509ff0 100644 --- a/pkg/kubeadm/master/apiclient.go +++ b/pkg/kubeadm/master/apiclient.go @@ -159,21 +159,24 @@ func findMyself(client *clientset.Clientset) (*api.Node, error) { return node, nil } -func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error { +func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, schedulable bool) error { n, err := findMyself(client) if err != nil { return err } n.ObjectMeta.Labels["kubeadm.alpha.kubernetes.io/role"] = "master" - taintsAnnotation, _ := json.Marshal([]api.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) - n.ObjectMeta.Annotations[api.TaintsAnnotationKey] = string(taintsAnnotation) + + if !schedulable { + taintsAnnotation, _ := json.Marshal([]api.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) + n.ObjectMeta.Annotations[api.TaintsAnnotationKey] = string(taintsAnnotation) + } if _, err := client.Nodes().Update(n); err != nil { if apierrs.IsConflict(err) { fmt.Println(" temporarily unable to update master node metadata due to conflict (will retry)") time.Sleep(500 * time.Millisecond) - attemptToUpdateMasterRoleLabelsAndTaints(client) + attemptToUpdateMasterRoleLabelsAndTaints(client, schedulable) } else { return err } @@ -182,8 +185,8 @@ func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error return nil } -func UpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error { - err := attemptToUpdateMasterRoleLabelsAndTaints(client) +func UpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, schedulable bool) error { + err := attemptToUpdateMasterRoleLabelsAndTaints(client, schedulable) if err != nil { return fmt.Errorf(" failed to update master node - %s", err) } From cab23e202e8b34bf1d3547e4652736aa4d02c2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Thu, 15 Sep 2016 17:40:42 +0300 Subject: [PATCH 13/32] Various improvements for kubeadm. Removed the user command, as it's too little time for implementing that. Now it's possible to use multiple arches. --- cmd/kubeadm/app/kubeadm.go | 11 ++---- pkg/kubeadm/cmd/cmd.go | 1 - pkg/kubeadm/cmd/manual.go | 3 +- pkg/kubeadm/cmd/user.go | 34 ----------------- pkg/kubeadm/images/images.go | 12 +++--- pkg/kubeadm/master/addons.go | 25 ++++++++---- pkg/kubeadm/master/manifests.go | 68 +++++++++++++++++++-------------- pkg/kubeadm/master/pki.go | 6 --- pkg/kubeadm/node/csr.go | 11 +++--- pkg/kubeadm/util/kubeconfig.go | 6 +-- pkg/kubeadm/util/tokens.go | 6 +-- 11 files changed, 82 insertions(+), 101 deletions(-) delete mode 100644 pkg/kubeadm/cmd/user.go diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 06b0a786b31..6b180b7e085 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -19,7 +19,6 @@ package app import ( "fmt" "os" - "path" "strings" "github.com/spf13/pflag" @@ -34,14 +33,12 @@ var CommandLine *pflag.FlagSet // TODO(phase2) use componentconfig // we need some params for testing etc, let's keep these hidden for now func getEnvParams() map[string]string { - globalPrefix := os.Getenv("KUBE_PREFIX_ALL") - if globalPrefix == "" { - globalPrefix = "/etc/kubernetes" - } envParams := map[string]string{ - "prefix": globalPrefix, - "host_pki_path": path.Join(globalPrefix, "pki"), + // TODO(phase1): Mode prefix and host_pki_path to another place as constants, and use them everywhere + // Right now they're used here and there, but not consequently + "kubernetes_dir": "/etc/kubernetes", + "host_pki_path": "/etc/kubernetes/pki", "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index dda31030e9a..04b51034edb 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -90,7 +90,6 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env cmds.AddCommand(NewCmdInit(out, s)) cmds.AddCommand(NewCmdJoin(out, s)) - cmds.AddCommand(NewCmdUser(out, s)) cmds.AddCommand(NewCmdManual(out, s)) return cmds diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index a95601931ca..2b397a47a84 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -120,9 +120,8 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, `(optional) DNS name to advertise, in case you have configured one yourself.`, ) - _, defaultServicesCIDR, _ := net.ParseCIDR("100.64.0.0/12") cmd.PersistentFlags().IPNetVar( - &s.InitFlags.Services.CIDR, "service-cidr", *defaultServicesCIDR, + &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, ) cmd.PersistentFlags().StringVar( diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go deleted file mode 100644 index 6beda1bf40e..00000000000 --- a/pkg/kubeadm/cmd/user.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "io" - - "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" -) - -func NewCmdUser(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - cmd := &cobra.Command{ - Use: "user", - Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap - Run: func(cmd *cobra.Command, args []string) { - }, - } - return cmd -} diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go index e1f5f48c95a..f699b382b1c 100644 --- a/pkg/kubeadm/images/images.go +++ b/pkg/kubeadm/images/images.go @@ -35,13 +35,15 @@ const ( gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" - kubeVersion = "v1.4.0-beta.6" kubeDnsVersion = "1.7" dnsmasqVersion = "1.3" exechealthzVersion = "1.1" ) +// TODO(phase1): Make this configurable + default to a v1.4 value fetched from: https://storage.googleapis.com/kubernetes-release/release/stable.txt +var DefaultKubeVersion = "v1.4.0-beta.6" + func GetCoreImage(image string, overrideImage string) string { if overrideImage != "" { return overrideImage @@ -49,10 +51,10 @@ func GetCoreImage(image string, overrideImage string) string { return map[string]string{ KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), - KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, DefaultKubeVersion), + KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, DefaultKubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, DefaultKubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, DefaultKubeVersion), }[image] } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index 362d6112411..44aa05ec5c1 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -19,6 +19,7 @@ package master import ( "fmt" "path" + "runtime" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" @@ -28,11 +29,14 @@ import ( ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) - -func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { +// TODO(phase1+): kube-proxy should be a daemonset, three different daemonsets should not be here +func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig, architecture string) api.PodSpec { privilegedTrue := true return api.PodSpec{ SecurityContext: &api.PodSecurityContext{HostNetwork: true}, + NodeSelector: map[string]string{ + "beta.kubernetes.io/arch": architecture, + }, Containers: []api.Container{{ Name: kubeProxy, Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]), @@ -65,7 +69,7 @@ func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { { Name: "kubeconfig", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["prefix"], "kubelet.conf")}, + HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["kubernetes_dir"], "kubelet.conf")}, }, }, { @@ -101,6 +105,9 @@ func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { ) return api.PodSpec{ + NodeSelector: map[string]string{ + "beta.kubernetes.io/arch": runtime.GOARCH, + }, Containers: []api.Container{ // DNS server { @@ -223,11 +230,15 @@ func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, er } func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset) error { - kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(s)) - SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) + arches := [3]string{"amd64", "arm", "arm64"} - if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { - return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + for _, arch := range arches { + kubeProxyDaemonSet := NewDaemonSet(kubeProxy + "-" + arch, createKubeProxyPodSpec(s, arch)) + SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) + + if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { + return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + } } fmt.Println(" created essential addon: kube-proxy") diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 2766e90d389..d545575d7af 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -36,7 +36,8 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - DefaultClusterName = "--cluster-name=kubernetes" + DefaultClusterName = "kubernetes" + DefaultCloudConfigPath = "/etc/kubernetes/cloud-config.json" etcd = "etcd" apiServer = "apiserver" @@ -57,12 +58,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume etcd: componentPod(api.Container{ - Command: []string{ - "/usr/local/bin/etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/etcd/data", - }, + Command: getComponentCommand(etcd, s), VolumeMounts: []api.VolumeMount{etcdVolumeMount()}, Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), @@ -74,18 +70,18 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(apiServer, s), - VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), - }, pkiVolume(s)), + }, k8sVolume(s)), kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(controllerManager, s), - VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), - }, pkiVolume(s)), + }, k8sVolume(s)), kubeScheduler: componentPod(api.Container{ Name: kubeScheduler, Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]), @@ -95,7 +91,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { }), } - manifestsPath := path.Join(s.EnvParams["prefix"], "manifests") + manifestsPath := path.Join(s.EnvParams["kubernetes_dir"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) } @@ -128,19 +124,19 @@ func etcdVolumeMount() api.VolumeMount { } } -func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume { +func k8sVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_pki_path"]}, + HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["kubernetes_dir"]}, }, } } -func pkiVolumeMount() api.VolumeMount { +func k8sVolumeMount() api.VolumeMount { return api.VolumeMount{ Name: "pki", - MountPath: "/etc/kubernetes/pki", + MountPath: "/etc/kubernetes/", ReadOnly: true, } } @@ -187,32 +183,43 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { } func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { - baseFalgs := map[string][]string{ + // TODO: make a global constant of this + pki_dir := "/etc/kubernetes/pki" + + baseFlags := map[string][]string{ + etcd: []string{ + "/usr/local/bin/etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/etcd/data", + }, apiServer: []string{ "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), - "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--client-ca-file=/etc/kubernetes/pki/ca.pem", - "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", - "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--service-account-key-file=" + pki_dir + "/apiserver-key.pem", + "--client-ca-file=" + pki_dir + "/ca.pem", + "--tls-cert-file=" + pki_dir + "/apiserver.pem", + "--tls-private-key-file=" + pki_dir + "/apiserver-key.pem", + "--token-auth-file=" + pki_dir + "/tokens.csv", "--secure-port=443", "--allow-privileged", - "--token-auth-file=/etc/kubernetes/pki/tokens.csv", }, controllerManager: []string{ + // TODO: consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", - DefaultClusterName, - "--root-ca-file=/etc/kubernetes/pki/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", - "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--cluster-name=" + DefaultClusterName, + "--root-ca-file=" + pki_dir + "/ca.pem", + "--service-account-private-key-file=" + pki_dir + "/apiserver-key.pem", + "--cluster-signing-cert-file=" + pki_dir + "/ca.pem", + "--cluster-signing-key-file=" + pki_dir + "/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), }, scheduler: []string{ + // TODO: consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", }, @@ -226,10 +233,15 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command } command = append(command, s.EnvParams["component_loglevel"]) - command = append(command, baseFalgs[component]...) + command = append(command, baseFlags[component]...) if component == controllerManager && s.InitFlags.CloudProvider != "" { command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) + + // Only append the --cloud-config option if there's a such file + if _, err := os.Stat(DefaultCloudConfigPath); err == nil { + command = append(command, "--cloud-config=" + DefaultCloudConfigPath) + } } return diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index 19c8ccac927..a28189c10ae 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -27,12 +27,6 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -/* -func errorf(f string, err error, vargs ...string) error { - return fmt.Errorf(" %s [%s]", fmt.Sprintf(f, v...), err) -} -*/ - func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index e6d841dd600..28fb584e56e 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "strings" + "os" "k8s.io/kubernetes/pkg/apis/certificates" unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" @@ -33,10 +34,6 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -func getNodeName() string { - return "TODO" -} - func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile) if err != nil { @@ -51,7 +48,11 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert // TODO try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) - nodeName := getNodeName() + // Try to fetch the hostname of the node + nodeName, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf(" failed to get node hostname [%v]", err) + } bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( *kubeadmutil.MakeClientConfigWithToken( diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 4fe56cf03f6..b7f3919fc0f 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -82,11 +82,11 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // start it again in that case). func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { - if err := os.MkdirAll(s.EnvParams["prefix"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["prefix"], err) + if err := os.MkdirAll(s.EnvParams["kubernetes_dir"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["kubernetes_dir"], err) } - filename := path.Join(s.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + filename := path.Join(s.EnvParams["kubernetes_dir"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile( filename, diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 202a37b2451..1e016088b4a 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -30,7 +30,7 @@ const ( TokenBytes = 8 ) -func randBytes(length int) ([]byte, string, error) { +func RandBytes(length int) ([]byte, string, error) { b := make([]byte, length) _, err := rand.Read(b) if err != nil { @@ -43,12 +43,12 @@ func randBytes(length int) ([]byte, string, error) { } func GenerateToken(s *kubeadmapi.KubeadmConfig) error { - _, tokenID, err := randBytes(TokenIDLen / 2) + _, tokenID, err := RandBytes(TokenIDLen / 2) if err != nil { return err } - tokenBytes, token, err := randBytes(TokenBytes) + tokenBytes, token, err := RandBytes(TokenBytes) if err != nil { return err } From a23239acfad0c493199b5f7584b0cb8efc3de427 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 15 Sep 2016 23:54:48 +0100 Subject: [PATCH 14/32] Capture an important to-do note --- pkg/kubeadm/cmd/cmd.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index 04b51034edb..8fbebe62a9d 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -77,6 +77,9 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env // be written to disc and write it all in one go at the end as we have a lot of // crapy little files written from different parts of this code; this could also // be useful for testing + // by having this model we can allow users to create some files before `kubeadm init` runs, e.g. PKI assets, we + // would then be able to look at files users has given an diff or validate if those are sane, we could also warn + // if any of the files had been deprecated s := new(kubeadmapi.KubeadmConfig) s.EnvParams = envParams From ca8a7a2c1cd51a12862af1e4646ccbdbb6152839 Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Fri, 16 Sep 2016 08:54:09 +0000 Subject: [PATCH 15/32] Fix Command for etcd Pod getComponentCommand method is too hyperkube specific to be used for etcd Pod. --- pkg/kubeadm/master/manifests.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index d545575d7af..a81f1eb7934 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -58,7 +58,12 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume etcd: componentPod(api.Container{ - Command: getComponentCommand(etcd, s), + Command: []string{ + "/usr/local/bin/etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/etcd/data", + }, VolumeMounts: []api.VolumeMount{etcdVolumeMount()}, Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), @@ -187,12 +192,6 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command pki_dir := "/etc/kubernetes/pki" baseFlags := map[string][]string{ - etcd: []string{ - "/usr/local/bin/etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/etcd/data", - }, apiServer: []string{ "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", From a42ad6a913e1ef1e288ab2f7de4826d85a5b738b Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Sun, 18 Sep 2016 12:25:42 +0100 Subject: [PATCH 16/32] Move `pkg/kubadm` to `cmd/kubeadm/app`, remove `cmd/manual.go` --- {pkg => cmd}/kubeadm/README.md | 0 {pkg/kubeadm => cmd/kubeadm/app}/api/types.go | 0 {pkg/kubeadm => cmd/kubeadm/app}/cmd/cmd.go | 7 +- {pkg/kubeadm => cmd/kubeadm/app}/cmd/init.go | 6 +- {pkg/kubeadm => cmd/kubeadm/app}/cmd/join.go | 6 +- .../kubeadm/app}/images/images.go | 0 cmd/kubeadm/app/kubeadm.go | 4 +- .../kubeadm/app}/master/addons.go | 4 +- .../kubeadm/app}/master/apiclient.go | 0 .../kubeadm/app}/master/discovery.go | 2 +- .../kubeadm/app}/master/kubeconfig.go | 4 +- .../kubeadm/app}/master/manifests.go | 4 +- .../kubeadm => cmd/kubeadm/app}/master/pki.go | 2 +- .../kubeadm/app}/master/tokens.go | 4 +- {pkg/kubeadm => cmd/kubeadm/app}/node/csr.go | 4 +- .../kubeadm/app}/node/discovery.go | 2 +- .../kubeadm/app}/util/kubeconfig.go | 2 +- .../kubeadm/app}/util/tokens.go | 2 +- pkg/kubeadm/.gitignore | 1 - pkg/kubeadm/cmd/manual.go | 250 ------------------ 20 files changed, 24 insertions(+), 280 deletions(-) rename {pkg => cmd}/kubeadm/README.md (100%) rename {pkg/kubeadm => cmd/kubeadm/app}/api/types.go (100%) rename {pkg/kubeadm => cmd/kubeadm/app}/cmd/cmd.go (95%) rename {pkg/kubeadm => cmd/kubeadm/app}/cmd/init.go (96%) rename {pkg/kubeadm => cmd/kubeadm/app}/cmd/join.go (94%) rename {pkg/kubeadm => cmd/kubeadm/app}/images/images.go (100%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/addons.go (98%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/apiclient.go (100%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/discovery.go (98%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/kubeconfig.go (94%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/manifests.go (98%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/pki.go (99%) rename {pkg/kubeadm => cmd/kubeadm/app}/master/tokens.go (94%) rename {pkg/kubeadm => cmd/kubeadm/app}/node/csr.go (97%) rename {pkg/kubeadm => cmd/kubeadm/app}/node/discovery.go (98%) rename {pkg/kubeadm => cmd/kubeadm/app}/util/kubeconfig.go (98%) rename {pkg/kubeadm => cmd/kubeadm/app}/util/tokens.go (97%) delete mode 100644 pkg/kubeadm/.gitignore delete mode 100644 pkg/kubeadm/cmd/manual.go diff --git a/pkg/kubeadm/README.md b/cmd/kubeadm/README.md similarity index 100% rename from pkg/kubeadm/README.md rename to cmd/kubeadm/README.md diff --git a/pkg/kubeadm/api/types.go b/cmd/kubeadm/app/api/types.go similarity index 100% rename from pkg/kubeadm/api/types.go rename to cmd/kubeadm/app/api/types.go diff --git a/pkg/kubeadm/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go similarity index 95% rename from pkg/kubeadm/cmd/cmd.go rename to cmd/kubeadm/app/cmd/cmd.go index 8fbebe62a9d..83fbca97ce5 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -25,7 +25,7 @@ import ( cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/flag" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, envParams map[string]string) *cobra.Command { @@ -84,16 +84,11 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env s := new(kubeadmapi.KubeadmConfig) s.EnvParams = envParams - //s.InitFlags, s.JoinFlags = new(kubeadmapi.InitFlags), new(kubeadmapi.JoinFlags) - - //s.ManualFlags = new(kubeadmapi.ManualFlags) - cmds.ResetFlags() cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) cmds.AddCommand(NewCmdInit(out, s)) cmds.AddCommand(NewCmdJoin(out, s)) - cmds.AddCommand(NewCmdManual(out, s)) return cmds } diff --git a/pkg/kubeadm/cmd/init.go b/cmd/kubeadm/app/cmd/init.go similarity index 96% rename from pkg/kubeadm/cmd/init.go rename to cmd/kubeadm/app/cmd/init.go index 691ef9d3e32..ce3e95f20de 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -24,9 +24,9 @@ import ( "github.com/renstrom/dedent" "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" netutil "k8s.io/kubernetes/pkg/util/net" ) diff --git a/pkg/kubeadm/cmd/join.go b/cmd/kubeadm/app/cmd/join.go similarity index 94% rename from pkg/kubeadm/cmd/join.go rename to cmd/kubeadm/app/cmd/join.go index c83f6663069..b8f60a17663 100644 --- a/pkg/kubeadm/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -24,9 +24,9 @@ import ( "github.com/renstrom/dedent" "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubenode "k8s.io/kubernetes/pkg/kubeadm/node" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubenode "k8s.io/kubernetes/cmd/kubeadm/app/node" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) diff --git a/pkg/kubeadm/images/images.go b/cmd/kubeadm/app/images/images.go similarity index 100% rename from pkg/kubeadm/images/images.go rename to cmd/kubeadm/app/images/images.go diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 6b180b7e085..a6580e3c59c 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/pflag" - "k8s.io/kubernetes/pkg/kubeadm/cmd" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/logs" ) @@ -37,7 +37,7 @@ func getEnvParams() map[string]string { envParams := map[string]string{ // TODO(phase1): Mode prefix and host_pki_path to another place as constants, and use them everywhere // Right now they're used here and there, but not consequently - "kubernetes_dir": "/etc/kubernetes", + "kubernetes_dir": "/etc/kubernetes", "host_pki_path": "/etc/kubernetes/pki", "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", diff --git a/pkg/kubeadm/master/addons.go b/cmd/kubeadm/app/master/addons.go similarity index 98% rename from pkg/kubeadm/master/addons.go rename to cmd/kubeadm/app/master/addons.go index 44aa05ec5c1..5e395052c70 100644 --- a/pkg/kubeadm/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -24,8 +24,8 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - "k8s.io/kubernetes/pkg/kubeadm/images" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + "k8s.io/kubernetes/cmd/kubeadm/app/images" ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) diff --git a/pkg/kubeadm/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go similarity index 100% rename from pkg/kubeadm/master/apiclient.go rename to cmd/kubeadm/app/master/apiclient.go diff --git a/pkg/kubeadm/master/discovery.go b/cmd/kubeadm/app/master/discovery.go similarity index 98% rename from pkg/kubeadm/master/discovery.go rename to cmd/kubeadm/app/master/discovery.go index 4ae8dcf13e6..39ddb6e60be 100644 --- a/pkg/kubeadm/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -25,7 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/pkg/kubeadm/master/kubeconfig.go b/cmd/kubeadm/app/master/kubeconfig.go similarity index 94% rename from pkg/kubeadm/master/kubeconfig.go rename to cmd/kubeadm/app/master/kubeconfig.go index 3287ea7a703..d9efe351479 100644 --- a/pkg/kubeadm/master/kubeconfig.go +++ b/cmd/kubeadm/app/master/kubeconfig.go @@ -23,8 +23,8 @@ import ( // TODO: "k8s.io/client-go/client/tools/clientcmd/api" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/pkg/kubeadm/master/manifests.go b/cmd/kubeadm/app/master/manifests.go similarity index 98% rename from pkg/kubeadm/master/manifests.go rename to cmd/kubeadm/app/master/manifests.go index a81f1eb7934..b08adda1e86 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -26,8 +26,8 @@ import ( "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/unversioned" api "k8s.io/kubernetes/pkg/api/v1" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - "k8s.io/kubernetes/pkg/kubeadm/images" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + "k8s.io/kubernetes/cmd/kubeadm/app/images" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/intstr" ) diff --git a/pkg/kubeadm/master/pki.go b/cmd/kubeadm/app/master/pki.go similarity index 99% rename from pkg/kubeadm/master/pki.go rename to cmd/kubeadm/app/master/pki.go index a28189c10ae..e904d41673c 100644 --- a/pkg/kubeadm/master/pki.go +++ b/cmd/kubeadm/app/master/pki.go @@ -22,7 +22,7 @@ import ( "fmt" "path" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/pkg/kubeadm/master/tokens.go b/cmd/kubeadm/app/master/tokens.go similarity index 94% rename from pkg/kubeadm/master/tokens.go rename to cmd/kubeadm/app/master/tokens.go index f741105d588..8f0cfd1c5ee 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/cmd/kubeadm/app/master/tokens.go @@ -22,8 +22,8 @@ import ( "os" "path" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/uuid" ) diff --git a/pkg/kubeadm/node/csr.go b/cmd/kubeadm/app/node/csr.go similarity index 97% rename from pkg/kubeadm/node/csr.go rename to cmd/kubeadm/app/node/csr.go index 28fb584e56e..012bd573d0f 100644 --- a/pkg/kubeadm/node/csr.go +++ b/cmd/kubeadm/app/node/csr.go @@ -28,8 +28,8 @@ import ( "k8s.io/kubernetes/pkg/client/typed/discovery" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/kubelet/util/csr" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/pkg/kubeadm/node/discovery.go b/cmd/kubeadm/app/node/discovery.go similarity index 98% rename from pkg/kubeadm/node/discovery.go rename to cmd/kubeadm/app/node/discovery.go index 8ec04b10d17..1f837fb2316 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/cmd/kubeadm/app/node/discovery.go @@ -25,7 +25,7 @@ import ( jose "github.com/square/go-jose" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { diff --git a/pkg/kubeadm/util/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig.go similarity index 98% rename from pkg/kubeadm/util/kubeconfig.go rename to cmd/kubeadm/app/util/kubeconfig.go index b7f3919fc0f..e419a3a8336 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig.go @@ -24,7 +24,7 @@ import ( // TODO: "k8s.io/client-go/client/tools/clientcmd/api" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) func CreateBasicClientConfig(clusterName string, serverURL string, caCert []byte) *clientcmdapi.Config { diff --git a/pkg/kubeadm/util/tokens.go b/cmd/kubeadm/app/util/tokens.go similarity index 97% rename from pkg/kubeadm/util/tokens.go rename to cmd/kubeadm/app/util/tokens.go index 1e016088b4a..44bc4e69f05 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/cmd/kubeadm/app/util/tokens.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) const ( diff --git a/pkg/kubeadm/.gitignore b/pkg/kubeadm/.gitignore deleted file mode 100644 index a8a20d9f1ba..00000000000 --- a/pkg/kubeadm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -kubeadm diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go deleted file mode 100644 index 2b397a47a84..00000000000 --- a/pkg/kubeadm/cmd/manual.go +++ /dev/null @@ -1,250 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "io" - "net" - - "github.com/renstrom/dedent" - "github.com/spf13/cobra" - - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" - kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" - kubenode "k8s.io/kubernetes/pkg/kubeadm/node" - kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - netutil "k8s.io/kubernetes/pkg/util/net" -) - -var ( - manual_init_done_msgf = dedent.Dedent(` - Master initialization complete: - - * Static pods written and kubelet's kubeconfig written. - * Kubelet should start soon. Try 'systemctl restart kubelet' - or equivalent if it doesn't. - - CA cert is written to: - /etc/kubernetes/pki/ca.pem. - - **Please copy this file (scp, rsync or through other means) to - all your nodes and then run on them**: - - kubeadm manual bootstrap join-node --ca-cert-file \ - --token %s --api-server-urls https://%s:443/ - `) - manual_join_done_msgf = dedent.Dedent(` - Node join complete: - * Certificate signing request sent to master and response - received. - * Kubelet informed of new secure connection details. - - Run 'kubectl get nodes' on the master to see this node join. - `) -) - -// TODO --token here becomes `s.Secrets.BearerToken` and not `s.Secrets.GivenToken` -// may be we should make it the same and ask user to pass dot-separated tokens -// in any of the modes; we could also enable discovery API in the manual mode just -// as well, there is no reason we shouldn't let user mix and match modes, unless -// it is too difficult to support - -func NewCmdManual(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - cmd := &cobra.Command{ - Use: "manual", - Short: "Advanced, less-automated functionality, for power users.", - // TODO put example usage in the Long description here - } - cmd.AddCommand(NewCmdManualBootstrap(out, s)) - return cmd -} - -func NewCmdManualBootstrap(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - cmd := &cobra.Command{ - Use: "bootstrap", - Short: "Manually bootstrap a cluster 'out-of-band'", - Long: dedent.Dedent(` - Manually bootstrap a cluster 'out-of-band', by generating and distributing a CA - certificate to all your servers and specifying and (list of) API server URLs. - `), - Run: func(cmd *cobra.Command, args []string) { - }, - } - cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, s)) - cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, s)) - - return cmd -} - -func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - advertiseAddrs := &[]string{} - cmd := &cobra.Command{ - Use: "init-master", - Short: "Manually bootstrap a master 'out-of-band'", - Long: dedent.Dedent(` - Manually bootstrap a master 'out-of-band'. - Will create TLS certificates and set up static pods for Kubernetes master - components. - `), - Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapInitMaster(out, cmd, args, s, advertiseAddrs) - cmdutil.CheckErr(err) - }, - } - - cmd.PersistentFlags().StringVar( - &s.Secrets.BearerToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, - ) - cmd.PersistentFlags().StringSliceVar( - advertiseAddrs, "api-advertise-addr", nil, - `(optional) IP address to advertise, in case autodetection fails.`, - ) - cmd.PersistentFlags().StringSliceVar( - &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, - `(optional) DNS name to advertise, in case you have configured one yourself.`, - ) - cmd.PersistentFlags().IPNetVar( - &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, - `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, - ) - cmd.PersistentFlags().StringVar( - &s.InitFlags.Services.DNSDomain, "service-dns-domain", "cluster.local", - `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, - ) - cmd.PersistentFlags().BoolVar( - &s.InitFlags.Schedulable, "schedule-workload", false, - `(optional) allow to schedule workload to the node`, - ) - - return cmd -} - -func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { - // Auto-detect the IP - if len(*advertiseAddrs) == 0 { - // TODO(phase1+) perhaps we could actually grab eth0 and eth1 - ip, err := netutil.ChooseHostInterface() - if err != nil { - return err - } - s.InitFlags.API.AdvertiseAddrs = []net.IP{ip} - } else { - for _, i := range *advertiseAddrs { - addr := net.ParseIP(i) - if addr == nil { - return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) - } - s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) - } - } - - if err := kubemaster.CreateTokenAuthFile(s); err != nil { - return err - } - if err := kubemaster.WriteStaticPodManifests(s); err != nil { - return err - } - caKey, caCert, err := kubemaster.CreatePKIAssets(s) - if err != nil { - return err - } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert) - if err != nil { - return err - } - for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { - return err - } - } - - // TODO we have most of cmd/init functionality here, except for `CreateDiscoveryDeploymentAndSecret()` - // it may be a good idea to just merge the two commands into one, and it's something we have started talking - // about, the only question is where disco service should be an opt-out... - - client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"]) - if err != nil { - return err - } - - if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, s.Schedulable); err != nil { - return err - } - - if err := kubemaster.CreateEssentialAddons(s, client); err != nil { - return err - } - - // TODO use templates to reference struct fields directly as order of args is fragile - fmt.Fprintf(out, manual_init_done_msgf, - s.Secrets.BearerToken, - s.InitFlags.API.AdvertiseAddrs[0].String(), - ) - return nil -} - -func NewCmdManualBootstrapJoinNode(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - cmd := &cobra.Command{ - Use: "join-node", - Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", - - Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapJoinNode(out, cmd, args, s) - cmdutil.CheckErr(err) - }, - } - cmd.PersistentFlags().StringVarP(&s.ManualFlags.CaCertFile, "ca-cert-file", "", "", - `Path to a CA cert file in PEM format. The same CA cert must be distributed to - all servers.`) - cmd.PersistentFlags().StringVarP(&s.ManualFlags.ApiServerURLs, "api-server-urls", "", "", - `Comma separated list of API server URLs. Typically this might be just - https://:8080/`) - cmd.PersistentFlags().StringVarP(&s.ManualFlags.BearerToken, "token", "", "", - `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) - - return cmd -} - -func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { - if s.ManualFlags.CaCertFile == "" { - fmt.Fprintf(out, "Must specify --ca-cert-file (see --help)\n") - return nil - } - - if s.ManualFlags.ApiServerURLs == "" { - fmt.Fprintf(out, "Must specify --api-server-urls (see --help)\n") - return nil - } - - kubeconfig, err := kubenode.PerformTLSBootstrapFromConfig(s) - if err != nil { - fmt.Fprintf(out, "Failed to perform TLS bootstrap: %s\n", err) - return err - } - - err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig) - if err != nil { - fmt.Fprintf(out, "Unable to write config for node:\n%s\n", err) - return err - } - - fmt.Fprintf(out, manual_join_done_msgf) - return nil -} From 26aa32d32b01c8ab833bcaeef40d7c80d904200d Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Mon, 19 Sep 2016 16:11:05 +0100 Subject: [PATCH 17/32] Reviewed help text, fix typos, go {fmt,vet,lint}. --- cmd/kubeadm/README.md | 49 ++++++++++++++++++++-------- cmd/kubeadm/app/api/types.go | 23 +++++-------- cmd/kubeadm/app/cmd/cmd.go | 9 +++-- cmd/kubeadm/app/cmd/init.go | 28 ++++++++-------- cmd/kubeadm/app/cmd/join.go | 15 +++++---- cmd/kubeadm/app/images/images.go | 14 ++++---- cmd/kubeadm/app/kubeadm.go | 3 -- cmd/kubeadm/app/master/addons.go | 11 ++++--- cmd/kubeadm/app/master/discovery.go | 2 +- cmd/kubeadm/app/master/kubeconfig.go | 2 +- cmd/kubeadm/app/master/manifests.go | 34 ++++++++++--------- cmd/kubeadm/app/master/pki.go | 8 ++--- cmd/kubeadm/app/node/csr.go | 17 +++------- cmd/kubeadm/app/node/discovery.go | 2 +- cmd/kubeadm/app/util/kubeconfig.go | 2 +- 15 files changed, 115 insertions(+), 104 deletions(-) diff --git a/cmd/kubeadm/README.md b/cmd/kubeadm/README.md index a43cde6a899..1b0d43c8319 100644 --- a/cmd/kubeadm/README.md +++ b/cmd/kubeadm/README.md @@ -5,42 +5,63 @@ ### `kubeadm init` It's usually enough to run `kubeadm init`, but in some case you might like to override the -default behaviour. +default behaviour. The flags used for said purpose are described below. -- `--token=` +- `--token=` -By default, a token is generated, but if you are to automate cluster deployment, you want to +By default, a token is generated, but if you are to automate cluster deployment, you will want to set the token ahead of time. Read the docs for more information on the token format. -- `--api-advertise-addr=` (multiple values allowed) -- `--api-external-dns-name=` (multiple values allowed) +- `--api-advertise-addresses=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) +- `--api-external-dns-names=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) -By default, `kubeadm` will auto detect IP address and use that to generate API server certificates. -If you would like to access the API via any external IPs and/or DNS, which it might not be able -to detect, you can use `--api-advertise-addr` and `--api-external-dns-name` to add multiple -different IP addresses and DNS names. +By default, `kubeadm` will auto detect IP addresses and use that to generate API server certificates. +If you would like to access the API via any external IPs and/or hostnames, which it might not be able +to detect, you can use `--api-advertise-addresses` and `--api-external-dns-names` to add multiple +different IP addresses and hostnames (DNS). + +- `--service-cidr=` (default: "100.64.0.0/12") + +By default, `kubeadm` sets `100.64.0.0/12` as the subnet for services. This means when a service is created, its cluster IP, if not manually specified, +will be automatically assigned from the services subnet. If you would like to set a different one, use `--service-cidr`. -- `--service-cidr=` (default: "100.64/12") - `--service-dns-domain=` (default: "cluster.local") -- `--use-hyperkube=` (default: "false") +By default, `kubeadm` sets `cluster.local` as the cluster DNS domain. If you would like to set a different one, use `--service-dns-domain`. + +- `--schedule-workload=` (default: "false") + +By default, `kubeadm` sets the master node kubelet as non-schedulable for workloads. This means the master node won't run your pods. If you want to change that, +use `--schedule-workload=true`. + +- `--cloud-provider=` + +By default, `kubeadm` doesn't perform auto-detection of the current cloud provider. If you want to specify it, use `--cloud-provider`. Possible values are +the ones supported by controller-manager, namely `"aws"`, `"azure"`, `"cloudstack"`, `"gce"`, `"mesos"`, `"openstack"`, `"ovirt"`, `"rackspace"`, `"vsphere"`. ***TODO(phase1+)*** -- `--api-bind-addr=` +- `--api-bind-address=` - `--api-bind-port=` ***TODO(phase2)*** - `--api-bind-loopback-unsecure=` -***TODO(pahse2)*** - - `--prefer-private-network=` - `--prefer-public-network=` ### `kubeadm join` +`kubeadm join` has one mandatory flag, the token used to secure cluster bootstrap, and one mandatory argument, the master IP address. +Here's an example on how to use it: + +`kubeadm join --token=the_secret_token 192.168.1.1` + +- `--token=` + +By default, when `kubeadm init` runs, a token is generated and revealed in the output. That's the token you should use here. + # User Experience Considerations > ***TODO*** _Move this into the design document diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index c6229a78d9c..5ff3638d81f 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -20,10 +20,10 @@ import ( "net" ) +// KubeadmConfig TODO add description type KubeadmConfig struct { InitFlags JoinFlags - ManualFlags Secrets struct { GivenToken string // dot-separated `.` set by the user TokenID string // optional on master side, will be generated if not specified @@ -33,19 +33,20 @@ type KubeadmConfig struct { EnvParams map[string]string // TODO(phase2) this is likely to be come componentconfig } -// TODO(phase2) should we add validatin funcs on these structs? +// TODO(phase2) should we add validation functions for these structs? +// InitFlags holds values for "kubeadm init" command flags. type InitFlags struct { API struct { - AdvertiseAddrs []net.IP - ExternalDNSName []string + AdvertiseAddrs []net.IP + ExternalDNSNames []string } Services struct { CIDR net.IPNet DNSDomain string } CloudProvider string - Schedulable bool + Schedulable bool } const ( @@ -77,20 +78,14 @@ func init() { } } +// JoinFlags holds values for "kubeadm join" command flags. type JoinFlags struct { MasterAddrs []net.IP } -// TODO(phase1?) we haven't decided whether manual sub commands should get merged into main commands... -type ManualFlags struct { - ApiServerURLs string // comma separated - CaCertFile string - BearerToken string // set based on Token - ListenIP net.IP // optional IP for master to listen on, rather than autodetect -} - +// ClusterInfo TODO add description type ClusterInfo struct { - // TODO(pahse1?) this may become simply `api.Config` + // TODO(phase1?) this may become simply `api.Config` CertificateAuthorities []string `json:"certificateAuthorities"` Endpoints []string `json:"endpoints"` } diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 83fbca97ce5..1fb97bc023c 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -31,9 +31,9 @@ import ( func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, envParams map[string]string) *cobra.Command { cmds := &cobra.Command{ Use: "kubeadm", - Short: "kubeadm: bootstrap a secure kubernetes cluster easily.", + Short: "kubeadm: easily bootstrap a secure Kubernetes cluster.", Long: dedent.Dedent(` - kubeadm: bootstrap a secure kubernetes cluster easily. + kubeadm: easily bootstrap a secure Kubernetes cluster. ┌──────────────────────────────────────────────────────────┐ │ KUBEADM IS ALPHA, DO NOT USE IT FOR PRODUCTION CLUSTERS! │ @@ -51,14 +51,13 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env ┌──────────────────────────────────────────────────────────┐ │ On the first machine │ ├──────────────────────────────────────────────────────────┤ - │ master# kubeadm init master │ - │ Your token is: │ + │ master# kubeadm init │ └──────────────────────────────────────────────────────────┘ ┌──────────────────────────────────────────────────────────┐ │ On the second machine │ ├──────────────────────────────────────────────────────────┤ - │ node# kubeadm join node --token= │ + │ node# kubeadm join --token= │ └──────────────────────────────────────────────────────────┘ You can then repeat the second step on as many other machines as you like. diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ce3e95f20de..72775ee6e9e 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -32,20 +32,21 @@ import ( ) var ( - init_done_msgf = dedent.Dedent(` + initDoneMsgf = dedent.Dedent(` Kubernetes master initialised successfully! - You can connect any number of nodes by running: + You can now join any number of machines by running the following on each node: kubeadm join --token %s %s `) ) +// NewCmdInit returns "kubeadm init" command. func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { advertiseAddrs := &[]string{} cmd := &cobra.Command{ Use: "init", - Short: "Run this on the first server you deploy onto.", + Short: "Run this on the first machine.", Run: func(cmd *cobra.Command, args []string) { err := RunInit(out, cmd, args, s, advertiseAddrs) cmdutil.CheckErr(err) // TODO(phase1+) append alpha warning with bugs URL etc @@ -54,28 +55,28 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd.PersistentFlags().StringVar( &s.Secrets.GivenToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, + `(optional) Shared secret used to secure cluster bootstrap. If none is provided, one will be generated for you.`, ) cmd.PersistentFlags().StringSliceVar( - advertiseAddrs, "api-advertise-addr", []string{}, - `(optional) IP address to advertise, in case autodetection fails.`, + advertiseAddrs, "api-advertise-addresses", []string{}, + `(optional) The IP addresses to advertise, in case autodetection fails.`, ) cmd.PersistentFlags().StringSliceVar( - &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, - `(optional) DNS name to advertise, in case you have configured one yourself.`, + &s.InitFlags.API.ExternalDNSNames, "api-external-dns-names", []string{}, + `(optional) The DNS names to advertise, in case you have configured them yourself.`, ) cmd.PersistentFlags().IPNetVar( &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, - `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + `(optional) use alternative range of IP address for service VIPs, e.g. "10.16.0.0/12"`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, - `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, + `(optional) use alternative domain for services, e.g. "myorg.internal"`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.CloudProvider, "cloud-provider", "", - `(optional) enable cloud proiver features (external load-balancers, storage, etc)`, + `(optional) enable a specific cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, ) cmd.PersistentFlags().BoolVar( &s.InitFlags.Schedulable, "schedule-workload", false, @@ -85,6 +86,7 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { return cmd } +// RunInit executes master node provisioning, including certificates, needed static pod manifests, etc. func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { // Auto-detect the IP if len(*advertiseAddrs) == 0 { @@ -98,7 +100,7 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub for _, i := range *advertiseAddrs { addr := net.ParseIP(i) if addr == nil { - return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) + return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addresses="+i) } s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) } @@ -152,7 +154,7 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub } // TODO(phase1+) use templates to reference struct fields directly as order of args is fragile - fmt.Fprintf(out, init_done_msgf, + fmt.Fprintf(out, initDoneMsgf, s.Secrets.GivenToken, s.InitFlags.API.AdvertiseAddrs[0].String(), ) diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index b8f60a17663..ecf7bde5d00 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -31,35 +31,36 @@ import ( ) var ( - join_done_msgf = dedent.Dedent(` + joinDoneMsgf = dedent.Dedent(` Node join complete: * Certificate signing request sent to master and response received. * Kubelet informed of new secure connection details. - Run 'kubectl get nodes' on the master to see this node join. + Run 'kubectl get nodes' on the master to see this machine join. `) ) +// NewCmdJoin returns "kubeadm join" command. func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "join", - Short: "Run this on other servers to join an existing cluster.", + Short: "Run this on any machine you wish to join an existing cluster.", Run: func(cmd *cobra.Command, args []string) { err := RunJoin(out, cmd, args, s) cmdutil.CheckErr(err) }, } - // TODO this should become `kubeadm join --token=<...> ` cmd.PersistentFlags().StringVarP( &s.Secrets.GivenToken, "token", "", "", - `Shared secret used to secure bootstrap. Must match output of 'init-master'.`, + `Shared secret used to secure bootstrap. Must match the output of 'kubeadm init'.`, ) return cmd } +// RunJoin executes worked node provisioning and tries to join an existing cluster. func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { // TODO this we are missing args from the help text, there should be a way to tell cobra about it if len(args) == 0 { @@ -68,7 +69,7 @@ func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub for _, i := range args { addr := net.ParseIP(i) // TODO(phase1+) should allow resolvable names too if addr == nil { - return fmt.Errorf(" failed parse argument (%q) as an IP address", i) + return fmt.Errorf(" failed to parse argument (%q) as an IP address", i) } s.JoinFlags.MasterAddrs = append(s.JoinFlags.MasterAddrs, addr) } @@ -91,6 +92,6 @@ func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub return err } - fmt.Fprintf(out, join_done_msgf) + fmt.Fprintf(out, joinDoneMsgf) return nil } diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index f699b382b1c..438d462d7ba 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -24,19 +24,19 @@ import ( const ( KubeEtcdImage = "etcd" - KubeApiServerImage = "apiserver" + KubeAPIServerImage = "apiserver" KubeControllerManagerImage = "controller-manager" KubeSchedulerImage = "scheduler" KubeProxyImage = "proxy" - KubeDnsImage = "kube-dns" - KubeDnsmasqImage = "dnsmasq" + KubeDNSImage = "kube-dns" + KubeDNSmasqImage = "dnsmasq" KubeExechealthzImage = "exechealthz" gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" - kubeDnsVersion = "1.7" + kubeDNSVersion = "1.7" dnsmasqVersion = "1.3" exechealthzVersion = "1.1" ) @@ -51,7 +51,7 @@ func GetCoreImage(image string, overrideImage string) string { return map[string]string{ KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, DefaultKubeVersion), + KubeAPIServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, DefaultKubeVersion), KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, DefaultKubeVersion), KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, DefaultKubeVersion), KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, DefaultKubeVersion), @@ -60,8 +60,8 @@ func GetCoreImage(image string, overrideImage string) string { func GetAddonImage(image string) string { return map[string]string{ - KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDnsVersion), - KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), + KubeDNSImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion), + KubeDNSmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), }[image] } diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index a6580e3c59c..6a6b92feb60 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -28,8 +28,6 @@ import ( "k8s.io/kubernetes/pkg/util/logs" ) -var CommandLine *pflag.FlagSet - // TODO(phase2) use componentconfig // we need some params for testing etc, let's keep these hidden for now func getEnvParams() map[string]string { @@ -56,7 +54,6 @@ func getEnvParams() map[string]string { } func Run() error { - CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) logs.InitLogs() defer logs.FlushLogs() diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index 5e395052c70..be2d6abed0d 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -21,14 +21,15 @@ import ( "path" "runtime" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + "k8s.io/kubernetes/cmd/kubeadm/app/images" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" - "k8s.io/kubernetes/cmd/kubeadm/app/images" ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) + // TODO(phase1+): kube-proxy should be a daemonset, three different daemonsets should not be here func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig, architecture string) api.PodSpec { privilegedTrue := true @@ -112,7 +113,7 @@ func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { // DNS server { Name: "kube-dns", - Image: images.GetAddonImage(images.KubeDnsImage), + Image: images.GetAddonImage(images.KubeDNSImage), Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -164,7 +165,7 @@ func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { // dnsmasq { Name: "dnsmasq", - Image: images.GetAddonImage(images.KubeDnsmasqImage), + Image: images.GetAddonImage(images.KubeDNSmasqImage), Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -233,7 +234,7 @@ func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Client arches := [3]string{"amd64", "arm", "arm64"} for _, arch := range arches { - kubeProxyDaemonSet := NewDaemonSet(kubeProxy + "-" + arch, createKubeProxyPodSpec(s, arch)) + kubeProxyDaemonSet := NewDaemonSet(kubeProxy+"-"+arch, createKubeProxyPodSpec(s, arch)) SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 39ddb6e60be..25ffd827bc2 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -22,10 +22,10 @@ import ( "encoding/json" "fmt" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/cmd/kubeadm/app/master/kubeconfig.go b/cmd/kubeadm/app/master/kubeconfig.go index d9efe351479..3ad4c3be554 100644 --- a/cmd/kubeadm/app/master/kubeconfig.go +++ b/cmd/kubeadm/app/master/kubeconfig.go @@ -22,9 +22,9 @@ import ( "fmt" // TODO: "k8s.io/client-go/client/tools/clientcmd/api" - clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" certutil "k8s.io/kubernetes/pkg/util/cert" ) diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index b08adda1e86..8c77af88cbb 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -23,11 +23,11 @@ import ( "os" "path" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + "k8s.io/kubernetes/cmd/kubeadm/app/images" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/unversioned" api "k8s.io/kubernetes/pkg/api/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" - "k8s.io/kubernetes/cmd/kubeadm/app/images" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -36,7 +36,7 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - DefaultClusterName = "kubernetes" + DefaultClusterName = "kubernetes" DefaultCloudConfigPath = "/etc/kubernetes/cloud-config.json" etcd = "etcd" @@ -54,6 +54,8 @@ const ( // //E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' +// WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk +// where kubelet will pick and schedule them. func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume @@ -73,7 +75,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { // TODO bind-mount certs in kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, - Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]), + Image: images.GetCoreImage(images.KubeAPIServerImage, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(apiServer, s), VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), @@ -113,6 +115,8 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { return nil } +// etcdVolume returns an host-path volume for storing etcd data. +// By using a host-path, the data will survive pod restart. func etcdVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "etcd", @@ -189,7 +193,7 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { // TODO: make a global constant of this - pki_dir := "/etc/kubernetes/pki" + pkiDir := "/etc/kubernetes/pki" baseFlags := map[string][]string{ apiServer: []string{ @@ -197,11 +201,11 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), - "--service-account-key-file=" + pki_dir + "/apiserver-key.pem", - "--client-ca-file=" + pki_dir + "/ca.pem", - "--tls-cert-file=" + pki_dir + "/apiserver.pem", - "--tls-private-key-file=" + pki_dir + "/apiserver-key.pem", - "--token-auth-file=" + pki_dir + "/tokens.csv", + "--service-account-key-file=" + pkiDir + "/apiserver-key.pem", + "--client-ca-file=" + pkiDir + "/ca.pem", + "--tls-cert-file=" + pkiDir + "/apiserver.pem", + "--tls-private-key-file=" + pkiDir + "/apiserver-key.pem", + "--token-auth-file=" + pkiDir + "/tokens.csv", "--secure-port=443", "--allow-privileged", }, @@ -210,10 +214,10 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--leader-elect", "--master=127.0.0.1:8080", "--cluster-name=" + DefaultClusterName, - "--root-ca-file=" + pki_dir + "/ca.pem", - "--service-account-private-key-file=" + pki_dir + "/apiserver-key.pem", - "--cluster-signing-cert-file=" + pki_dir + "/ca.pem", - "--cluster-signing-key-file=" + pki_dir + "/ca-key.pem", + "--root-ca-file=" + pkiDir + "/ca.pem", + "--service-account-private-key-file=" + pkiDir + "/apiserver-key.pem", + "--cluster-signing-cert-file=" + pkiDir + "/ca.pem", + "--cluster-signing-key-file=" + pkiDir + "/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), }, @@ -239,7 +243,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command // Only append the --cloud-config option if there's a such file if _, err := os.Stat(DefaultCloudConfigPath); err == nil { - command = append(command, "--cloud-config=" + DefaultCloudConfigPath) + command = append(command, "--cloud-config="+DefaultCloudConfigPath) } } diff --git a/cmd/kubeadm/app/master/pki.go b/cmd/kubeadm/app/master/pki.go index e904d41673c..584ea9d1d25 100644 --- a/cmd/kubeadm/app/master/pki.go +++ b/cmd/kubeadm/app/master/pki.go @@ -60,7 +60,7 @@ func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 1) if err != nil { - return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]") + return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]", &s.InitFlags.Services.CIDR, err) } altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP) @@ -117,7 +117,7 @@ func writeKeysAndCert(pkiPath string, name string, key *rsa.PrivateKey, cert *x5 if cert != nil { if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil { - return fmt.Errorf("unable to write certificate file (%q) [%s]", err) + return fmt.Errorf("unable to write certificate file (%q) [%s]", certificatePath, err) } } @@ -142,8 +142,8 @@ func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certif altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...) } - if len(s.InitFlags.API.ExternalDNSName) > 0 { - altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSName...) + if len(s.InitFlags.API.ExternalDNSNames) > 0 { + altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSNames...) } pkiPath := path.Join(s.EnvParams["host_pki_path"]) diff --git a/cmd/kubeadm/app/node/csr.go b/cmd/kubeadm/app/node/csr.go index 012bd573d0f..06a9453dfad 100644 --- a/cmd/kubeadm/app/node/csr.go +++ b/cmd/kubeadm/app/node/csr.go @@ -19,31 +19,22 @@ package node import ( "fmt" "io/ioutil" - "strings" "os" + "strings" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/apis/certificates" unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/client/typed/discovery" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/kubelet/util/csr" certutil "k8s.io/kubernetes/pkg/util/cert" ) -func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { - caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile) - if err != nil { - return nil, fmt.Errorf(" failed to load CA certificate [%s]", err) - } - - return PerformTLSBootstrap(s, strings.Split(s.ManualFlags.ApiServerURLs, ",")[0], caCert) -} - -// Create a restful client for doing the certificate signing request. +// PerformTLSBootstrap creates a RESTful client in order to execute certificate signing request. func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { // TODO try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) diff --git a/cmd/kubeadm/app/node/discovery.go b/cmd/kubeadm/app/node/discovery.go index 1f837fb2316..42a1794a840 100644 --- a/cmd/kubeadm/app/node/discovery.go +++ b/cmd/kubeadm/app/node/discovery.go @@ -24,8 +24,8 @@ import ( "net/http" jose "github.com/square/go-jose" - clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" ) func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { diff --git a/cmd/kubeadm/app/util/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig.go index e419a3a8336..3ca4d087211 100644 --- a/cmd/kubeadm/app/util/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig.go @@ -22,9 +22,9 @@ import ( "path" // TODO: "k8s.io/client-go/client/tools/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) func CreateBasicClientConfig(clusterName string, serverURL string, caCert []byte) *clientcmdapi.Config { From 389cb2c7cdcd984bd949bbc3fb565c5990a10b8b Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Tue, 20 Sep 2016 11:37:02 +0100 Subject: [PATCH 18/32] Add support for external and optionally secured etcd cluster. --- cmd/kubeadm/app/api/types.go | 6 +++ cmd/kubeadm/app/cmd/init.go | 19 +++++++ cmd/kubeadm/app/master/manifests.go | 83 ++++++++++++++++++++++------- 3 files changed, 88 insertions(+), 20 deletions(-) diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index 5ff3638d81f..ffdd22e4317 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -40,6 +40,12 @@ type InitFlags struct { API struct { AdvertiseAddrs []net.IP ExternalDNSNames []string + Etcd struct { + ExternalEndpoints []string + ExternalCAFile string + ExternalCertFile string + ExternalKeyFile string + } } Services struct { CIDR net.IPNet diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 72775ee6e9e..c8bfda9a452 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -83,6 +83,25 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { `(optional) allow to schedule workload to the node`, ) + // TODO (phase1+) @errordeveloper make the flags below not show up in --help but rather on --advanced-help + + cmd.PersistentFlags().StringSliceVar( + &s.InitFlags.API.Etcd.ExternalEndpoints, "external-etcd-endpoints", []string{}, + `(optional) etcd endpoints to use, in case you have an external cluster.`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.API.Etcd.ExternalCAFile, "external-etcd-cafile", "", + `(optional) etcd certificate authority certificate file."`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.API.Etcd.ExternalCertFile, "external-etcd-certfile", "", + `(optional) etcd client certificate file."`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.API.Etcd.ExternalKeyFile, "external-etcd-keyfile", "", + `(optional) etcd client key file."`, + ) + return cmd } diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 8c77af88cbb..00bb9521eb9 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path" + "strings" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" "k8s.io/kubernetes/cmd/kubeadm/app/images" @@ -57,30 +58,38 @@ const ( // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk // where kubelet will pick and schedule them. func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { + // Placeholder for kube-apiserver pod spec command + apiServerCommand := getComponentCommand(apiServer, s) + + // Check if the user decided to use an external etcd cluster + if len(s.InitFlags.API.Etcd.ExternalEndpoints) > 0 { + arg := fmt.Sprintf("--etcd-servers=%s", strings.Join(s.InitFlags.API.Etcd.ExternalEndpoints, ",")) + apiServerCommand = append(apiServerCommand, arg) + } else { + apiServerCommand = append(apiServerCommand, "--etcd-servers=http://127.0.0.1:2379") + } + + // Is etcd secured? + if s.InitFlags.API.Etcd.ExternalCAFile != "" { + etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", s.InitFlags.API.Etcd.ExternalCAFile) + apiServerCommand = append(apiServerCommand, etcdCAFileArg) + } + if s.InitFlags.API.Etcd.ExternalCertFile != "" && s.InitFlags.API.Etcd.ExternalKeyFile != "" { + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", s.InitFlags.API.Etcd.ExternalCertFile) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", s.InitFlags.API.Etcd.ExternalKeyFile) + apiServerCommand = append(apiServerCommand, etcdClientFileArg, etcdKeyFileArg) + } + + // Prepare static pod specs staticPodSpecs := map[string]api.Pod{ - // TODO this needs a volume - etcd: componentPod(api.Container{ - Command: []string{ - "/usr/local/bin/etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/etcd/data", - }, - VolumeMounts: []api.VolumeMount{etcdVolumeMount()}, - Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), - LivenessProbe: componentProbe(2379, "/health"), - Name: etcd, - Resources: componentResources("200m"), - }, etcdVolume(s)), - // TODO bind-mount certs in kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, s.EnvParams["hyperkube_image"]), - Command: getComponentCommand(apiServer, s), - VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, + Command: apiServerCommand, + VolumeMounts: []api.VolumeMount{certsVolumeMount(), k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), - }, k8sVolume(s)), + }, certsVolume(s), k8sVolume(s)), kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), @@ -98,6 +107,23 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { }), } + // Add etcd static pod spec only if external etcd is not configured + if len(s.InitFlags.API.Etcd.ExternalEndpoints) == 0 { + staticPodSpecs[etcd] = componentPod(api.Container{ + Name: etcd, + Command: []string{ + "etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/etcd/data", + }, + VolumeMounts: []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(), k8sVolumeMount()}, + Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), + LivenessProbe: componentProbe(2379, "/health"), + Resources: componentResources("200m"), + }, certsVolume(s), etcdVolume(s), k8sVolume(s)) + } + manifestsPath := path.Join(s.EnvParams["kubernetes_dir"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) @@ -115,8 +141,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { return nil } -// etcdVolume returns an host-path volume for storing etcd data. -// By using a host-path, the data will survive pod restart. +// etcdVolume exposes a path on the host in order to guarantee data survival during reboot. func etcdVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "etcd", @@ -133,6 +158,24 @@ func etcdVolumeMount() api.VolumeMount { } } +// certsVolume exposes host SSL certificates to pod containers. +func certsVolume(s *kubeadmapi.KubeadmConfig) api.Volume { + return api.Volume{ + Name: "certs", + VolumeSource: api.VolumeSource{ + // TODO make path configurable + HostPath: &api.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + }, + } +} + +func certsVolumeMount() api.VolumeMount { + return api.VolumeMount{ + Name: "certs", + MountPath: "/etc/ssl/certs", + } +} + func k8sVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", From 38b53e31f3a33ea3f7f12bd8de41d96e0918a6bf Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Tue, 20 Sep 2016 12:13:47 +0100 Subject: [PATCH 19/32] Before declaring success, require that the discovery deployment has at least one active pod. --- cmd/kubeadm/app/master/discovery.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 25ffd827bc2..e36fbe4cc93 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -21,12 +21,14 @@ import ( "encoding/hex" "encoding/json" "fmt" + "time" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" certutil "k8s.io/kubernetes/pkg/util/cert" + "k8s.io/kubernetes/pkg/util/wait" ) type kubeDiscovery struct { @@ -35,7 +37,7 @@ type kubeDiscovery struct { } const ( - kubeDiscoverynName = "kube-discovery" + kubeDiscoveryName = "kube-discovery" kubeDiscoverySecretName = "clusterinfo" ) @@ -71,7 +73,7 @@ func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { // TODO update this when #31307 is resolved SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ - Name: kubeDiscoverynName, + Name: kubeDiscoveryName, Image: s.EnvParams["discovery_image"], Command: []string{"/usr/bin/kube-discovery"}, VolumeMounts: []api.VolumeMount{{ @@ -96,7 +98,7 @@ func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery { kd := kubeDiscovery{ - Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(s)), + Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(s)), Secret: &api.Secret{ ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, Type: api.SecretTypeOpaque, @@ -114,15 +116,27 @@ func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *cli kd := newKubeDiscovery(s, caCert) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { - return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoverynName, err) + return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoveryName, err) } if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil { return fmt.Errorf(" failed to create %q secret [%s]", kubeDiscoverySecretName, err) } - fmt.Println(" created essential addon: kube-discovery") + fmt.Println(" created essential addon: kube-discovery, waiting for it to become ready") - // TODO we should probably wait for the pod to become ready + // wait for the pod to become ready + start := time.Now() + wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName) + if err != nil { + return false, nil + } + if d.Status.AvailableReplicas < 1 { + return false, nil + } + return true, nil + }) + fmt.Printf(" kube-discovery is ready after %f seconds\n", time.Since(start).Seconds()) return nil } From 9eeae34581b161301de3bc2b2dfed3dcd9e40f51 Mon Sep 17 00:00:00 2001 From: Atanas Mirchev Date: Mon, 19 Sep 2016 09:44:17 +0200 Subject: [PATCH 20/32] Add node CIDR allocation as an option to kubeadm. This is useful for users who are used to deploying with a flannel overlay network. --- cmd/kubeadm/app/api/types.go | 3 +++ cmd/kubeadm/app/cmd/init.go | 15 ++++++++++----- cmd/kubeadm/app/master/manifests.go | 19 +++++++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index ffdd22e4317..808f978cebb 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -51,6 +51,9 @@ type InitFlags struct { CIDR net.IPNet DNSDomain string } + PodNetwork struct { + CIDR net.IPNet + } CloudProvider string Schedulable bool } diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index c8bfda9a452..2eeb645e48b 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -65,22 +65,27 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { &s.InitFlags.API.ExternalDNSNames, "api-external-dns-names", []string{}, `(optional) The DNS names to advertise, in case you have configured them yourself.`, ) - cmd.PersistentFlags().IPNetVar( &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, - `(optional) use alternative range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + `(optional) Use alterantive range of IP address for service VIPs, defaults to `+ + kubeadmapi.DefaultServicesCIDRString, + ) + cmd.PersistentFlags().IPNetVar( + &s.InitFlags.PodNetwork.CIDR, "pod-network-cidr", net.IPNet{}, + `(optional) Specify range of IP addresses for the pod overlay network, e.g. 10.3.0.0/16.`+ + `If set, the control plane will automatically attempt to allocate Pod network CIDRs for every node.`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, - `(optional) use alternative domain for services, e.g. "myorg.internal"`, + `(optional) Use alternative domain for services, e.g. "myorg.internal"`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.CloudProvider, "cloud-provider", "", - `(optional) enable a specific cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, + `(optional) Enable a specific cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, ) cmd.PersistentFlags().BoolVar( &s.InitFlags.Schedulable, "schedule-workload", false, - `(optional) allow to schedule workload to the node`, + `(optional) Allow to schedule workload to the node`, ) // TODO (phase1+) @errordeveloper make the flags below not show up in --help but rather on --advanced-help diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 00bb9521eb9..738e157ffc7 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -262,7 +262,6 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--cluster-signing-cert-file=" + pkiDir + "/ca.pem", "--cluster-signing-key-file=" + pkiDir + "/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", - "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), }, scheduler: []string{ // TODO: consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world @@ -281,12 +280,20 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command command = append(command, s.EnvParams["component_loglevel"]) command = append(command, baseFlags[component]...) - if component == controllerManager && s.InitFlags.CloudProvider != "" { - command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) + if component == controllerManager { + if s.InitFlags.CloudProvider != "" { + command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) - // Only append the --cloud-config option if there's a such file - if _, err := os.Stat(DefaultCloudConfigPath); err == nil { - command = append(command, "--cloud-config="+DefaultCloudConfigPath) + // Only append the --cloud-config option if there's a such file + if _, err := os.Stat(DefaultCloudConfigPath); err == nil { + command = append(command, "--cloud-config="+DefaultCloudConfigPath) + } + } + + if s.InitFlags.PodNetwork.CIDR.IP != nil { + // Let the controller-manager allocate Node CIDRs for the Pod overlay network. + // Each node will get a subspace of the address CIDR provided with --cluster-cidr. + command = append(command, "--allocate-node-cidrs=true", "--cluster-cidr="+s.InitFlags.PodNetwork.CIDR.String()) } } From 0f05ccb019638d83736ead573e5307482ced3c53 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Mon, 19 Sep 2016 20:05:14 +0100 Subject: [PATCH 21/32] Cleanup some low-hanging fruits and review TODOs --- cmd/kubeadm/README.md | 62 +++++++---------------------- cmd/kubeadm/app/api/types.go | 3 +- cmd/kubeadm/app/cmd/cmd.go | 6 +-- cmd/kubeadm/app/cmd/init.go | 15 +++++-- cmd/kubeadm/app/cmd/join.go | 2 +- cmd/kubeadm/app/images/images.go | 2 +- cmd/kubeadm/app/kubeadm.go | 9 ++++- cmd/kubeadm/app/master/apiclient.go | 13 +++--- cmd/kubeadm/app/master/discovery.go | 9 +---- cmd/kubeadm/app/master/manifests.go | 62 +++++++++++++---------------- cmd/kubeadm/app/master/pki.go | 2 +- cmd/kubeadm/app/node/csr.go | 30 ++++++-------- cmd/kubeadm/app/node/discovery.go | 5 +-- cmd/kubeadm/app/util/kubeconfig.go | 12 +----- cmd/kubeadm/app/util/tokens.go | 4 +- cmd/kubeadm/kubeadm.go | 4 +- 16 files changed, 96 insertions(+), 144 deletions(-) diff --git a/cmd/kubeadm/README.md b/cmd/kubeadm/README.md index 1b0d43c8319..ea0607f6a18 100644 --- a/cmd/kubeadm/README.md +++ b/cmd/kubeadm/README.md @@ -1,4 +1,4 @@ -# Kubernetes Cluster Boostrap Made Easy +# Kubernetes Cluster Bootstrap Made Easy ## Usage @@ -9,35 +9,35 @@ default behaviour. The flags used for said purpose are described below. - `--token=` -By default, a token is generated, but if you are to automate cluster deployment, you will want to -set the token ahead of time. Read the docs for more information on the token format. + By default, a token is generated, but if you are to automate cluster deployment, you will want to + set the token ahead of time. Read the docs for more information on the token format. - `--api-advertise-addresses=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) - `--api-external-dns-names=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) -By default, `kubeadm` will auto detect IP addresses and use that to generate API server certificates. -If you would like to access the API via any external IPs and/or hostnames, which it might not be able -to detect, you can use `--api-advertise-addresses` and `--api-external-dns-names` to add multiple -different IP addresses and hostnames (DNS). + By default, `kubeadm` will auto detect IP addresses and use that to generate API server certificates. + If you would like to access the API via any external IPs and/or hostnames, which it might not be able + to detect, you can use `--api-advertise-addresses` and `--api-external-dns-names` to add multiple + different IP addresses and hostnames (DNS). - `--service-cidr=` (default: "100.64.0.0/12") -By default, `kubeadm` sets `100.64.0.0/12` as the subnet for services. This means when a service is created, its cluster IP, if not manually specified, -will be automatically assigned from the services subnet. If you would like to set a different one, use `--service-cidr`. + By default, `kubeadm` sets `100.64.0.0/12` as the subnet for services. This means when a service is created, its cluster IP, if not manually specified, + will be automatically assigned from the services subnet. If you would like to set a different one, use `--service-cidr`. - `--service-dns-domain=` (default: "cluster.local") -By default, `kubeadm` sets `cluster.local` as the cluster DNS domain. If you would like to set a different one, use `--service-dns-domain`. + By default, `kubeadm` sets `cluster.local` as the cluster DNS domain. If you would like to set a different one, use `--service-dns-domain`. - `--schedule-workload=` (default: "false") -By default, `kubeadm` sets the master node kubelet as non-schedulable for workloads. This means the master node won't run your pods. If you want to change that, -use `--schedule-workload=true`. + By default, `kubeadm` sets the master node kubelet as non-schedulable for workloads. This means the master node won't run your pods. If you want to change that, + use `--schedule-workload=true`. - `--cloud-provider=` -By default, `kubeadm` doesn't perform auto-detection of the current cloud provider. If you want to specify it, use `--cloud-provider`. Possible values are -the ones supported by controller-manager, namely `"aws"`, `"azure"`, `"cloudstack"`, `"gce"`, `"mesos"`, `"openstack"`, `"ovirt"`, `"rackspace"`, `"vsphere"`. + By default, `kubeadm` doesn't perform auto-detection of the current cloud provider. If you want to specify it, use `--cloud-provider`. Possible values are + the ones supported by controller-manager, namely `"aws"`, `"azure"`, `"cloudstack"`, `"gce"`, `"mesos"`, `"openstack"`, `"ovirt"`, `"rackspace"`, `"vsphere"`. ***TODO(phase1+)*** @@ -61,37 +61,3 @@ Here's an example on how to use it: - `--token=` By default, when `kubeadm init` runs, a token is generated and revealed in the output. That's the token you should use here. - -# User Experience Considerations - -> ***TODO*** _Move this into the design document - -a) `kube-apiserver` will listen on `0.0.0.0:443` and `127.0.0.1:8080`, which is not configurable right now and make things a bit easier for the MVP -b) there is `kube-discovery`, which will listen on `0.0.0.0:9898` - - -from the point of view of `kubeadm init`, we need to have -a) a primary IP address as will be seen by the nodes and needs to go into the cert and `kube-discovery` configuration secret -b) some other names and addresses API server may be known by (e.g. external DNS and/or LB and/or NAT) - -from that perspective we don’t can assume default ports for now, but for all the address we really only care about two ports (i.e. 443 and 9898) - -we should make ports configurable and expose some way of making API server bind to a specific address/interface - -but I think for the MVP we need solve the issue with hardcode IPs and DNS names in the certs - -so it sounds rather simple enough to introduce `--api-advertise-addr=` and `--api-external-dns-name=`, and allowing multiple of those sounds also simple enough - -from the `kubeadm join` perspective, it cares about the two ports we mentioned, and we can make those configurable too - -but for now it’s easier to pass just the IP address - -plust it’s also require, so passing it without a named flag sounds convenient, and it’s something users are familiar with - -that’s what Consul does, what what Weave does, and now Docker SwarmKit does the same thing also (edited) - -flags will differ, as there are some Kubernetes-specifics aspects to what join does, but basic join semantics will remain _familiar_ - -if we do marry `kube-discovery` with the API, we might do `kubeadm join host:port`, as we’d end-up with a single port to care about - -but we haven’t yet diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index 808f978cebb..4152e1bfe1b 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -90,11 +90,12 @@ func init() { // JoinFlags holds values for "kubeadm join" command flags. type JoinFlags struct { MasterAddrs []net.IP + // TODO(phase1+) add manual mode flags here, e.g. RootCACertPath } // ClusterInfo TODO add description type ClusterInfo struct { - // TODO(phase1?) this may become simply `api.Config` + // TODO(phase1+) this may become simply `api.Config` CertificateAuthorities []string `json:"certificateAuthorities"` Endpoints []string `json:"endpoints"` } diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 1fb97bc023c..e46539719a9 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -22,10 +22,9 @@ import ( "github.com/renstrom/dedent" "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/flag" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, envParams map[string]string) *cobra.Command { @@ -66,9 +65,6 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env } // TODO(phase2+) figure out how to avoid running as root // - // TODO(phase1) also print the alpha warning when running any commands, as well as - // in the help text. - // // TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly // i.e. make it automation friendly // diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 2eeb645e48b..2e249f9280b 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -49,7 +49,7 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { Short: "Run this on the first machine.", Run: func(cmd *cobra.Command, args []string) { err := RunInit(out, cmd, args, s, advertiseAddrs) - cmdutil.CheckErr(err) // TODO(phase1+) append alpha warning with bugs URL etc + cmdutil.CheckErr(err) }, } @@ -72,8 +72,7 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { ) cmd.PersistentFlags().IPNetVar( &s.InitFlags.PodNetwork.CIDR, "pod-network-cidr", net.IPNet{}, - `(optional) Specify range of IP addresses for the pod overlay network, e.g. 10.3.0.0/16.`+ - `If set, the control plane will automatically attempt to allocate Pod network CIDRs for every node.`, + `(optional) Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, @@ -154,6 +153,16 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub if err != nil { return err } + + // kubeadm is responsible for writing the following kubeconfig file, which + // kubelet should be waiting for. Help user avoid foot-shooting by refusing to + // write a file that has already been written (the kubelet will be up and + // running in that case - they'd need to stop the kubelet, remove the file, and + // start it again in that case). + // TODO(phase1+) this is no longer the right place to guard agains foo-shooting, + // we need to decide how to handle existing files (it may be handy to support + // importing existing files, may be we could even make our command idempotant, + // or at least allow for external PKI and stuff) for name, kubeconfig := range kubeconfigs { if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { return err diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index ecf7bde5d00..1cba613f5ae 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -62,7 +62,7 @@ func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { // RunJoin executes worked node provisioning and tries to join an existing cluster. func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { - // TODO this we are missing args from the help text, there should be a way to tell cobra about it + // TODO(phase1+) this we are missing args from the help text, there should be a way to tell cobra about it if len(args) == 0 { return fmt.Errorf(" must specify master IP address (see --help)") } diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 438d462d7ba..a899a87cbf2 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -42,7 +42,7 @@ const ( ) // TODO(phase1): Make this configurable + default to a v1.4 value fetched from: https://storage.googleapis.com/kubernetes-release/release/stable.txt -var DefaultKubeVersion = "v1.4.0-beta.6" +var DefaultKubeVersion = "v1.4.0-beta.8" func GetCoreImage(image string, overrideImage string) string { if overrideImage != "" { diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 6a6b92feb60..1a288e800ca 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -21,6 +21,7 @@ import ( "os" "strings" + "github.com/renstrom/dedent" "github.com/spf13/pflag" "k8s.io/kubernetes/cmd/kubeadm/app/cmd" @@ -28,12 +29,18 @@ import ( "k8s.io/kubernetes/pkg/util/logs" ) +var AlphaWarningOnExit = dedent.Dedent(` + kubeadm: I am an alpha version, my authors welcome your feedback and bug reports + kubeadm: please create issue an using https://github.com/kubernetes/kubernetes/issues/new + kubeadm: and make sure to mention @kubernetes/sig-cluster-lifecycle. Thank you! +`) + // TODO(phase2) use componentconfig // we need some params for testing etc, let's keep these hidden for now func getEnvParams() map[string]string { envParams := map[string]string{ - // TODO(phase1): Mode prefix and host_pki_path to another place as constants, and use them everywhere + // TODO(phase1+): Mode prefix and host_pki_path to another place as constants, and use them everywhere // Right now they're used here and there, but not consequently "kubernetes_dir": "/etc/kubernetes", "host_pki_path": "/etc/kubernetes/pki", diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index 777ff509ff0..ed6cd99c02f 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -31,6 +31,8 @@ import ( "k8s.io/kubernetes/pkg/util/wait" ) +const apiCallRetryInterval = 500 * time.Millisecond + func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) { adminClientConfig, err := clientcmd.NewDefaultClientConfig( *adminConfig, @@ -50,12 +52,12 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli fmt.Println(" created API client, waiting for the control plane to become ready") start := time.Now() - wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { cs, err := client.ComponentStatuses().List(api.ListOptions{}) if err != nil { return false, nil } - // TODO revisit this when we implement HA + // TODO(phase2) must revisit this when we implement HA if len(cs.Items) < 3 { fmt.Println(" not all control plane components are ready yet") return false, nil @@ -75,14 +77,13 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli fmt.Println(" waiting for at least one node to register and become ready") start = time.Now() - wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { nodeList, err := client.Nodes().List(api.ListOptions{}) if err != nil { fmt.Println(" temporarily unable to list nodes (will retry)") return false, nil } if len(nodeList.Items) < 1 { - //fmt.Printf(" %d nodes have registered so far", len(nodeList.Items)) return false, nil } n := &nodeList.Items[0] @@ -146,7 +147,7 @@ func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) * } // It's safe to do this for alpha, as we don't have HA and there is no way we can get -// more then one node here (TODO find a way to determine owr own node name) +// more then one node here (TODO(phase1+) use os.Hostname) func findMyself(client *clientset.Clientset) (*api.Node, error) { nodeList, err := client.Nodes().List(api.ListOptions{}) if err != nil { @@ -175,7 +176,7 @@ func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, sched if _, err := client.Nodes().Update(n); err != nil { if apierrs.IsConflict(err) { fmt.Println(" temporarily unable to update master node metadata due to conflict (will retry)") - time.Sleep(500 * time.Millisecond) + time.Sleep(apiCallRetryInterval) attemptToUpdateMasterRoleLabelsAndTaints(client, schedulable) } else { return err diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index e36fbe4cc93..2afec6193a8 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -42,10 +42,6 @@ const ( ) func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte { - // TODO ListenIP is probably not the right now, although it's best we have right now - // if user provides a DNS name, or anything else, we should use that, may be it's really - // the list of all SANs (minus internal DNS names and service IP)? - var ( data = map[string][]byte{} endpointList = []string{} @@ -75,7 +71,7 @@ func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { Containers: []api.Container{{ Name: kubeDiscoveryName, Image: s.EnvParams["discovery_image"], - Command: []string{"/usr/bin/kube-discovery"}, + Command: []string{"/usr/local/bin/kube-discovery"}, VolumeMounts: []api.VolumeMount{{ Name: kubeDiscoverySecretName, MountPath: "/tmp/secret", // TODO use a shared constant @@ -124,9 +120,8 @@ func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *cli fmt.Println(" created essential addon: kube-discovery, waiting for it to become ready") - // wait for the pod to become ready start := time.Now() - wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName) if err != nil { return false, nil diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 738e157ffc7..247b8281a06 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -49,43 +49,18 @@ const ( kubeControllerManager = "kube-controller-manager" kubeScheduler = "kube-scheduler" kubeProxy = "kube-proxy" + pkiDir = "/etc/kubernetes/pki" ) -// TODO look into what this really means, scheduler prints it for some reason -// -//E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' - // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk // where kubelet will pick and schedule them. func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { - // Placeholder for kube-apiserver pod spec command - apiServerCommand := getComponentCommand(apiServer, s) - - // Check if the user decided to use an external etcd cluster - if len(s.InitFlags.API.Etcd.ExternalEndpoints) > 0 { - arg := fmt.Sprintf("--etcd-servers=%s", strings.Join(s.InitFlags.API.Etcd.ExternalEndpoints, ",")) - apiServerCommand = append(apiServerCommand, arg) - } else { - apiServerCommand = append(apiServerCommand, "--etcd-servers=http://127.0.0.1:2379") - } - - // Is etcd secured? - if s.InitFlags.API.Etcd.ExternalCAFile != "" { - etcdCAFileArg := fmt.Sprintf("--etcd-cafile=%s", s.InitFlags.API.Etcd.ExternalCAFile) - apiServerCommand = append(apiServerCommand, etcdCAFileArg) - } - if s.InitFlags.API.Etcd.ExternalCertFile != "" && s.InitFlags.API.Etcd.ExternalKeyFile != "" { - etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", s.InitFlags.API.Etcd.ExternalCertFile) - etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", s.InitFlags.API.Etcd.ExternalKeyFile) - apiServerCommand = append(apiServerCommand, etcdClientFileArg, etcdKeyFileArg) - } - // Prepare static pod specs staticPodSpecs := map[string]api.Pod{ kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, s.EnvParams["hyperkube_image"]), - Command: apiServerCommand, + Command: getComponentCommand(apiServer, s), VolumeMounts: []api.VolumeMount{certsVolumeMount(), k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), @@ -163,7 +138,7 @@ func certsVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "certs", VolumeSource: api.VolumeSource{ - // TODO make path configurable + // TODO(phase1+) make path configurable HostPath: &api.HostPathVolumeSource{Path: "/etc/ssl/certs"}, }, } @@ -235,9 +210,6 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { } func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { - // TODO: make a global constant of this - pkiDir := "/etc/kubernetes/pki" - baseFlags := map[string][]string{ apiServer: []string{ "--address=127.0.0.1", @@ -253,7 +225,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--allow-privileged", }, controllerManager: []string{ - // TODO: consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world + // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", "--cluster-name=" + DefaultClusterName, @@ -264,7 +236,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", }, scheduler: []string{ - // TODO: consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world + // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", }, @@ -280,19 +252,39 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command command = append(command, s.EnvParams["component_loglevel"]) command = append(command, baseFlags[component]...) + if component == apiServer { + // Check if the user decided to use an external etcd cluster + if len(s.InitFlags.API.Etcd.ExternalEndpoints) > 0 { + command = append(command, fmt.Sprintf("--etcd-servers=%s", strings.Join(s.InitFlags.API.Etcd.ExternalEndpoints, ","))) + } else { + command = append(command, "--etcd-servers=http://127.0.0.1:2379") + } + + // Is etcd secured? + if s.InitFlags.API.Etcd.ExternalCAFile != "" { + command = append(command, fmt.Sprintf("--etcd-cafile=%s", s.InitFlags.API.Etcd.ExternalCAFile)) + } + if s.InitFlags.API.Etcd.ExternalCertFile != "" && s.InitFlags.API.Etcd.ExternalKeyFile != "" { + etcdClientFileArg := fmt.Sprintf("--etcd-certfile=%s", s.InitFlags.API.Etcd.ExternalCertFile) + etcdKeyFileArg := fmt.Sprintf("--etcd-keyfile=%s", s.InitFlags.API.Etcd.ExternalKeyFile) + command = append(command, etcdClientFileArg, etcdKeyFileArg) + } + } + if component == controllerManager { if s.InitFlags.CloudProvider != "" { command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) // Only append the --cloud-config option if there's a such file + // TODO(phase1+) this won't work unless it's in one of the few directories we bind-mount if _, err := os.Stat(DefaultCloudConfigPath); err == nil { command = append(command, "--cloud-config="+DefaultCloudConfigPath) } } if s.InitFlags.PodNetwork.CIDR.IP != nil { - // Let the controller-manager allocate Node CIDRs for the Pod overlay network. - // Each node will get a subspace of the address CIDR provided with --cluster-cidr. + // Let the controller-manager allocate Node CIDRs for the Pod network. + // Each node will get a subspace of the address CIDR provided with --pod-network-cidr. command = append(command, "--allocate-node-cidrs=true", "--cluster-cidr="+s.InitFlags.PodNetwork.CIDR.String()) } } diff --git a/cmd/kubeadm/app/master/pki.go b/cmd/kubeadm/app/master/pki.go index 584ea9d1d25..e8ad7f479df 100644 --- a/cmd/kubeadm/app/master/pki.go +++ b/cmd/kubeadm/app/master/pki.go @@ -175,7 +175,7 @@ func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certif return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) } - // TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certiicates + // TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certificates fmt.Printf(" created keys and certificates in %q\n", pkiPath) return caKey, caCert, nil } diff --git a/cmd/kubeadm/app/node/csr.go b/cmd/kubeadm/app/node/csr.go index 06a9453dfad..8bd61e12451 100644 --- a/cmd/kubeadm/app/node/csr.go +++ b/cmd/kubeadm/app/node/csr.go @@ -18,9 +18,7 @@ package node import ( "fmt" - "io/ioutil" "os" - "strings" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -36,10 +34,9 @@ import ( // PerformTLSBootstrap creates a RESTful client in order to execute certificate signing request. func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { - // TODO try all the api servers until we find one that works + // TODO(phase1+) try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) - // Try to fetch the hostname of the node nodeName, err := os.Hostname() if err != nil { return nil, fmt.Errorf(" failed to get node hostname [%v]", err) @@ -55,18 +52,20 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) } - err = checkCertsAPI(bootstrapClientConfig) - - if err != nil { - return nil, fmt.Errorf(" API compatibility error [%s]", err) - } - client, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) if err != nil { return nil, fmt.Errorf(" failed to create API client [%s]", err) } csrClient := client.CertificateSigningRequests() + // TODO(phase1+) checkCertsAPI() has a side-effect of making first attempt of communicating with the API, + // we should _make it more explicit_ and have a user-settable _retry timeout_ to account for potential connectivity issues + // (for example user may be bringing up machines in parallel and for some reasons master is slow to boot) + + if err := checkCertsAPI(bootstrapClientConfig); err != nil { + return nil, fmt.Errorf(" fialed to proceed due to API compatibility issue - %s", err) + } + fmt.Println(" created API client to obtain unique certificate for this node, generating keys and certificate signing request") key, err := certutil.MakeEllipticPrivateKeyPEM() @@ -79,7 +78,7 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert return nil, fmt.Errorf(" failed to request signed certificate from the API server [%s]", err) } - // TODO print some basic info about the cert + // TODO(phase1+) print some basic info about the cert fmt.Println(" received signed certificate from the API server, generating kubelet configuration") finalConfig := kubeadmutil.MakeClientConfigWithCerts( @@ -110,14 +109,9 @@ func checkCertsAPI(config *restclient.Config) error { } version, err := discoveryClient.ServerVersion() - serverVersion := version.String() - if err != nil { - serverVersion = "N/A" + return fmt.Errorf("unable to obtain API version [%s]", err) } - // Due to changes in API namespaces for certificates - // https://github.com/kubernetes/kubernetes/pull/31887/ - // it is compatible only with versions released after v1.4.0-beta.0 - return fmt.Errorf("installed Kubernetes API server version \"%s\" does not support certificates signing request use v1.4.0 or newer", serverVersion) + return fmt.Errorf("API version %s does not support certificates API, use v1.4.0 or newer", version.String()) } diff --git a/cmd/kubeadm/app/node/discovery.go b/cmd/kubeadm/app/node/discovery.go index 42a1794a840..969557fca16 100644 --- a/cmd/kubeadm/app/node/discovery.go +++ b/cmd/kubeadm/app/node/discovery.go @@ -68,11 +68,10 @@ func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Conf return nil, fmt.Errorf(" cluster info object is invalid - no endpoint(s) and/or root CA certificate(s) found") } - // TODO print checksum of the CA certificate + // TODO(phase1+) print summary info about the CA certificate, along with the the checksum signature + // we also need an ability for the user to configure the client to validate recieved CA cert agains a checksum fmt.Printf(" cluster info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) - // TODO we need to configure the client to validate the server - // if it is signed by any of the returned certificates apiServer := clusterInfo.Endpoints[0] caCert := []byte(clusterInfo.CertificateAuthorities[0]) diff --git a/cmd/kubeadm/app/util/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig.go index 3ca4d087211..934e76c3dc7 100644 --- a/cmd/kubeadm/app/util/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig.go @@ -75,12 +75,6 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, return newConfig } -// kubeadm is responsible for writing the following kubeconfig file, which -// kubelet should be waiting for. Help user avoid foot-shooting by refusing to -// write a file that has already been written (the kubelet will be up and -// running in that case - they'd need to stop the kubelet, remove the file, and -// start it again in that case). - func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { if err := os.MkdirAll(s.EnvParams["kubernetes_dir"], 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["kubernetes_dir"], err) @@ -88,11 +82,7 @@ func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeco filename := path.Join(s.EnvParams["kubernetes_dir"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. - f, err := os.OpenFile( - filename, - os.O_CREATE|os.O_WRONLY|os.O_EXCL, - 0600, - ) + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) if err != nil { return fmt.Errorf(" failed to create %q, it already exists [%s]", filename, err) } diff --git a/cmd/kubeadm/app/util/tokens.go b/cmd/kubeadm/app/util/tokens.go index 44bc4e69f05..99da5e6557c 100644 --- a/cmd/kubeadm/app/util/tokens.go +++ b/cmd/kubeadm/app/util/tokens.go @@ -66,8 +66,8 @@ func UseGivenTokenIfValid(s *kubeadmapi.KubeadmConfig) (bool, error) { } fmt.Println(" validating provided token") givenToken := strings.Split(strings.ToLower(s.Secrets.GivenToken), ".") - // TODO print desired format - // TODO could also print more specific messages in each case + // TODO(phase1+) print desired format + // TODO(phase1+) could also print more specific messages in each case invalidErr := " provided token is invalid - %s" if len(givenToken) != 2 { return false, fmt.Errorf(invalidErr, "not in 2-part dot-separated format") diff --git a/cmd/kubeadm/kubeadm.go b/cmd/kubeadm/kubeadm.go index f61297fc8ff..902e46127da 100644 --- a/cmd/kubeadm/kubeadm.go +++ b/cmd/kubeadm/kubeadm.go @@ -17,14 +17,16 @@ limitations under the License. package main import ( + "fmt" "os" "k8s.io/kubernetes/cmd/kubeadm/app" ) -// TODO(phase1): check for root +// TODO(phase1+): check for root func main() { if err := app.Run(); err != nil { + fmt.Printf(app.AlphaWarningOnExit) os.Exit(1) } os.Exit(0) From 832d83efaa2dbb8afe17141a7392b0b9ec7aa300 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 21 Sep 2016 12:35:22 -0300 Subject: [PATCH 22/32] Allow etcd container to work with selinux. --- cmd/kubeadm/app/master/manifests.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 247b8281a06..ed90430c057 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -96,6 +96,15 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), Resources: componentResources("200m"), + SecurityContext: &api.SecurityContext{ + SELinuxOptions: &api.SELinuxOptions{ + // TODO: This implies our etcd container is not being restricted by + // SELinux. This is not optimal and would be nice to adjust in future + // so it can create and write /var/lib/etcd, but for now this avoids + // recommending setenforce 0 system-wide. + Type: "unconfined_t", + }, + }, }, certsVolume(s), etcdVolume(s), k8sVolume(s)) } From ab3b2d579fffbd4b9d6e5e2a306042666f866dc7 Mon Sep 17 00:00:00 2001 From: Atanas Mirchev Date: Thu, 22 Sep 2016 14:47:23 +0200 Subject: [PATCH 23/32] Fix package / struct naming after core refactoring. --- cmd/kubeadm/app/master/addons.go | 2 +- cmd/kubeadm/app/master/pki.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index be2d6abed0d..85e89c38ae7 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" + ipallocator "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) diff --git a/cmd/kubeadm/app/master/pki.go b/cmd/kubeadm/app/master/pki.go index e8ad7f479df..2cfed6d8b9e 100644 --- a/cmd/kubeadm/app/master/pki.go +++ b/cmd/kubeadm/app/master/pki.go @@ -23,7 +23,7 @@ import ( "path" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" - ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" + ipallocator "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" certutil "k8s.io/kubernetes/pkg/util/cert" ) @@ -33,7 +33,7 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { return nil, nil, fmt.Errorf("unable to create private key [%s]", err) } - config := certutil.CertConfig{ + config := certutil.Config{ CommonName: "kubernetes", } @@ -66,7 +66,7 @@ func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP) altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...) - config := certutil.CertConfig{ + config := certutil.Config{ CommonName: "kube-apiserver", AltNames: altNames, } @@ -84,7 +84,7 @@ func newClientKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*rsa. return nil, nil, fmt.Errorf("unable to create private key [%s]", err) } - config := certutil.CertConfig{ + config := certutil.Config{ CommonName: "kubernetes-admin", } cert, err := certutil.NewSignedCert(config, key, caCert, caKey) From 987da0186b84830fbf996ad27957bb0ef4c0f90d Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Fri, 23 Sep 2016 08:47:37 +0000 Subject: [PATCH 24/32] Add kubeadm flags to known-flags.txt --- hack/verify-flags/known-flags.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index a703e4d5316..43a7c40d091 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -9,6 +9,8 @@ all-namespaces allocate-node-cidrs allow-privileged allowed-not-ready-nodes +api-advertise-addresses +api-external-dns-names api-burst api-prefix api-rate @@ -176,6 +178,10 @@ experimental-keystone-url experimental-nvidia-gpus experimental-prefix experimental-runtime-integration-type +external-etcd-cafile +external-etcd-certfile +external-etcd-endpoints +external-etcd-keyfile external-hostname external-ip extra-peer-dirs @@ -385,6 +391,7 @@ pod-cidr pod-eviction-timeout pod-infra-container-image pod-manifest-path +pod-network-cidr pod-running pods-per-core policy-config-file @@ -448,6 +455,7 @@ runtime-config runtime-integration-type runtime-request-timeout save-config +schedule-workload scheduler-config scheduler-name schema-cache-dir @@ -462,6 +470,8 @@ service-account-lookup service-account-private-key-file service-address service-cluster-ip-range +service-cidr +service-dns-domain service-generator service-node-port-range service-node-ports From 0a68bb05eaa15f59c29930fec75492860a63af63 Mon Sep 17 00:00:00 2001 From: Evgeny L Date: Fri, 23 Sep 2016 10:47:05 +0000 Subject: [PATCH 25/32] Rename flag `--schedule-workload` to `--schedule-pods-here` for kubeadm init --- cmd/kubeadm/README.md | 4 ++-- cmd/kubeadm/app/cmd/init.go | 2 +- hack/verify-flags/known-flags.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/kubeadm/README.md b/cmd/kubeadm/README.md index ea0607f6a18..3b03a29a5ec 100644 --- a/cmd/kubeadm/README.md +++ b/cmd/kubeadm/README.md @@ -29,10 +29,10 @@ default behaviour. The flags used for said purpose are described below. By default, `kubeadm` sets `cluster.local` as the cluster DNS domain. If you would like to set a different one, use `--service-dns-domain`. -- `--schedule-workload=` (default: "false") +- `--schedule-pods-here=` (default: "false") By default, `kubeadm` sets the master node kubelet as non-schedulable for workloads. This means the master node won't run your pods. If you want to change that, - use `--schedule-workload=true`. + use `--schedule-pods-here=true`. - `--cloud-provider=` diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 2e249f9280b..83d0b02558d 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -83,7 +83,7 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { `(optional) Enable a specific cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, ) cmd.PersistentFlags().BoolVar( - &s.InitFlags.Schedulable, "schedule-workload", false, + &s.InitFlags.Schedulable, "schedule-pods-here", false, `(optional) Allow to schedule workload to the node`, ) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 43a7c40d091..1c98b734dcb 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -455,7 +455,7 @@ runtime-config runtime-integration-type runtime-request-timeout save-config -schedule-workload +schedule-pods-here scheduler-config scheduler-name schema-cache-dir From 5862ea6f385c53803176ea9d45a4bdbc5eb547da Mon Sep 17 00:00:00 2001 From: Atanas Mirchev Date: Fri, 23 Sep 2016 17:26:08 +0200 Subject: [PATCH 26/32] Fix boostrap token encoding bug during master init Currently the boostrap fails when a token is provided by the user on `master init` and works when the token is generated. This is because of a mismatch of how the token string in the kube-discovery secret is encoded. --- cmd/kubeadm/app/master/discovery.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 2afec6193a8..54401ef1ae6 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -18,7 +18,6 @@ package master import ( "crypto/x509" - "encoding/hex" "encoding/json" "fmt" "time" @@ -52,7 +51,7 @@ func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Cer endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String())) } - tokenMap[s.Secrets.TokenID] = hex.EncodeToString(s.Secrets.Token) + tokenMap[s.Secrets.TokenID] = s.Secrets.BearerToken data["endpoint-list.json"], _ = json.Marshal(endpointList) data["token-map.json"], _ = json.Marshal(tokenMap) From 37dab8017346dfd9c813e2501cc34eebf17c6527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 23 Sep 2016 16:40:38 +0300 Subject: [PATCH 27/32] Make the Kubernetes control plane version configurable --- cmd/kubeadm/app/api/types.go | 4 ++++ cmd/kubeadm/app/cmd/init.go | 5 ++++- cmd/kubeadm/app/cmd/join.go | 4 ++-- cmd/kubeadm/app/images/images.go | 15 +++++++-------- cmd/kubeadm/app/master/addons.go | 2 +- cmd/kubeadm/app/master/manifests.go | 8 ++++---- hack/verify-flags/known-flags.txt | 1 + 7 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index 4152e1bfe1b..f753be58037 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -54,6 +54,9 @@ type InitFlags struct { PodNetwork struct { CIDR net.IPNet } + Versions struct { + Kubernetes string + } CloudProvider string Schedulable bool } @@ -61,6 +64,7 @@ type InitFlags struct { const ( DefaultServiceDNSDomain = "cluster.local" DefaultServicesCIDRString = "100.64.0.0/12" + DefaultKubernetesVersion = "v1.4.0-beta.10" ) var ( diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 83d0b02558d..b4548bd202e 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -86,9 +86,12 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { &s.InitFlags.Schedulable, "schedule-pods-here", false, `(optional) Allow to schedule workload to the node`, ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.Versions.Kubernetes, "use-kubernetes-version", kubeadmapi.DefaultKubernetesVersion, + `(optional) Choose a specific Kubernetes version for the control plane`, + ) // TODO (phase1+) @errordeveloper make the flags below not show up in --help but rather on --advanced-help - cmd.PersistentFlags().StringSliceVar( &s.InitFlags.API.Etcd.ExternalEndpoints, "external-etcd-endpoints", []string{}, `(optional) etcd endpoints to use, in case you have an external cluster.`, diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 1cba613f5ae..a5fe58ab975 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -52,8 +52,8 @@ func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { }, } - cmd.PersistentFlags().StringVarP( - &s.Secrets.GivenToken, "token", "", "", + cmd.PersistentFlags().StringVar( + &s.Secrets.GivenToken, "token", "", `Shared secret used to secure bootstrap. Must match the output of 'kubeadm init'.`, ) diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index a899a87cbf2..bda982c0797 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -19,6 +19,8 @@ package images import ( "fmt" "runtime" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api" ) const ( @@ -41,20 +43,17 @@ const ( exechealthzVersion = "1.1" ) -// TODO(phase1): Make this configurable + default to a v1.4 value fetched from: https://storage.googleapis.com/kubernetes-release/release/stable.txt -var DefaultKubeVersion = "v1.4.0-beta.8" - -func GetCoreImage(image string, overrideImage string) string { +func GetCoreImage(image string, cfg *kubeadmapi.KubeadmConfig, overrideImage string) string { if overrideImage != "" { return overrideImage } return map[string]string{ KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeAPIServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, DefaultKubeVersion), - KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, DefaultKubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, DefaultKubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, DefaultKubeVersion), + KubeAPIServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, cfg.Versions.Kubernetes), + KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, cfg.Versions.Kubernetes), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, cfg.Versions.Kubernetes), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, cfg.Versions.Kubernetes), }[image] } diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index 85e89c38ae7..e95be511332 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -40,7 +40,7 @@ func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig, architecture string) ap }, Containers: []api.Container{{ Name: kubeProxy, - Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]), + Image: images.GetCoreImage(images.KubeProxyImage, s, s.EnvParams["hyperkube_image"]), Command: append(getComponentCommand("proxy", s), "--kubeconfig=/run/kubeconfig"), SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, VolumeMounts: []api.VolumeMount{ diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index ed90430c057..db82c7c30dd 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -59,7 +59,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, - Image: images.GetCoreImage(images.KubeAPIServerImage, s.EnvParams["hyperkube_image"]), + Image: images.GetCoreImage(images.KubeAPIServerImage, s, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(apiServer, s), VolumeMounts: []api.VolumeMount{certsVolumeMount(), k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), @@ -67,7 +67,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { }, certsVolume(s), k8sVolume(s)), kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, - Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), + Image: images.GetCoreImage(images.KubeControllerManagerImage, s, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(controllerManager, s), VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), @@ -75,7 +75,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { }, k8sVolume(s)), kubeScheduler: componentPod(api.Container{ Name: kubeScheduler, - Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]), + Image: images.GetCoreImage(images.KubeSchedulerImage, s, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(scheduler, s), LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), @@ -93,7 +93,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { "--data-dir=/var/etcd/data", }, VolumeMounts: []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(), k8sVolumeMount()}, - Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), + Image: images.GetCoreImage(images.KubeEtcdImage, s, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), Resources: componentResources("200m"), SecurityContext: &api.SecurityContext{ diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1c98b734dcb..7435cf1a08f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -532,6 +532,7 @@ update-period upgrade-image upgrade-target use-kubernetes-cluster-service +use-kubernetes-version user-whitelist verify-only volume-dir From d0e29789b07fb1642d99117fd84aaa9f66eb7578 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Fri, 23 Sep 2016 18:35:17 +0100 Subject: [PATCH 28/32] Fix sorting of linted packages and gofmt --- cmd/kubeadm/app/master/manifests.go | 8 ++++---- hack/.linted_packages | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index db82c7c30dd..24cfeca6afb 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -220,7 +220,7 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { baseFlags := map[string][]string{ - apiServer: []string{ + apiServer: { "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", @@ -233,7 +233,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--secure-port=443", "--allow-privileged", }, - controllerManager: []string{ + controllerManager: { // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", @@ -244,12 +244,12 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--cluster-signing-key-file=" + pkiDir + "/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", }, - scheduler: []string{ + scheduler: { // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", }, - proxy: []string{}, + proxy: {}, } if s.EnvParams["hyperkube_image"] != "" { diff --git a/hack/.linted_packages b/hack/.linted_packages index 3fd57c57f7e..caeb063b662 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -15,6 +15,7 @@ cmd/kube-discovery cmd/kube-dns cmd/kube-proxy cmd/kubeadm +cmd/kubeadm cmd/kubectl cmd/kubelet cmd/kubernetes-discovery @@ -243,4 +244,3 @@ test/integration/openshift test/soak/cauldron test/soak/serve_hostnames third_party/forked/golang/expansion -cmd/kubeadm From a023085a5fa0185b0e07bf82bcbac0443a4d5341 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Sat, 24 Sep 2016 14:40:10 +0100 Subject: [PATCH 29/32] Address comments in review --- cmd/kubeadm/README.md | 63 ---------------------------- cmd/kubeadm/app/api/types.go | 8 +++- cmd/kubeadm/app/cmd/init.go | 36 ++++++++-------- cmd/kubeadm/app/cmd/join.go | 4 +- cmd/kubeadm/app/master/addons.go | 14 +++---- cmd/kubeadm/app/master/apiclient.go | 9 ++-- cmd/kubeadm/app/master/discovery.go | 4 +- cmd/kubeadm/app/master/kubeconfig.go | 2 +- cmd/kubeadm/app/master/manifests.go | 9 ++-- cmd/kubeadm/app/master/pki.go | 47 ++++++++++----------- cmd/kubeadm/app/master/tokens.go | 8 ++-- cmd/kubeadm/app/node/csr.go | 16 +++---- cmd/kubeadm/app/node/discovery.go | 10 ++--- cmd/kubeadm/app/util/kubeconfig.go | 6 +-- 14 files changed, 91 insertions(+), 145 deletions(-) delete mode 100644 cmd/kubeadm/README.md diff --git a/cmd/kubeadm/README.md b/cmd/kubeadm/README.md deleted file mode 100644 index 3b03a29a5ec..00000000000 --- a/cmd/kubeadm/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Kubernetes Cluster Bootstrap Made Easy - -## Usage - -### `kubeadm init` - -It's usually enough to run `kubeadm init`, but in some case you might like to override the -default behaviour. The flags used for said purpose are described below. - -- `--token=` - - By default, a token is generated, but if you are to automate cluster deployment, you will want to - set the token ahead of time. Read the docs for more information on the token format. - -- `--api-advertise-addresses=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) -- `--api-external-dns-names=` (multiple values are allowed by having multiple flag declarations or multiple values separated by comma) - - By default, `kubeadm` will auto detect IP addresses and use that to generate API server certificates. - If you would like to access the API via any external IPs and/or hostnames, which it might not be able - to detect, you can use `--api-advertise-addresses` and `--api-external-dns-names` to add multiple - different IP addresses and hostnames (DNS). - -- `--service-cidr=` (default: "100.64.0.0/12") - - By default, `kubeadm` sets `100.64.0.0/12` as the subnet for services. This means when a service is created, its cluster IP, if not manually specified, - will be automatically assigned from the services subnet. If you would like to set a different one, use `--service-cidr`. - -- `--service-dns-domain=` (default: "cluster.local") - - By default, `kubeadm` sets `cluster.local` as the cluster DNS domain. If you would like to set a different one, use `--service-dns-domain`. - -- `--schedule-pods-here=` (default: "false") - - By default, `kubeadm` sets the master node kubelet as non-schedulable for workloads. This means the master node won't run your pods. If you want to change that, - use `--schedule-pods-here=true`. - -- `--cloud-provider=` - - By default, `kubeadm` doesn't perform auto-detection of the current cloud provider. If you want to specify it, use `--cloud-provider`. Possible values are - the ones supported by controller-manager, namely `"aws"`, `"azure"`, `"cloudstack"`, `"gce"`, `"mesos"`, `"openstack"`, `"ovirt"`, `"rackspace"`, `"vsphere"`. - -***TODO(phase1+)*** - -- `--api-bind-address=` -- `--api-bind-port=` - -***TODO(phase2)*** - -- `--api-bind-loopback-unsecure=` - -- `--prefer-private-network=` -- `--prefer-public-network=` - -### `kubeadm join` - -`kubeadm join` has one mandatory flag, the token used to secure cluster bootstrap, and one mandatory argument, the master IP address. -Here's an example on how to use it: - -`kubeadm join --token=the_secret_token 192.168.1.1` - -- `--token=` - -By default, when `kubeadm init` runs, a token is generated and revealed in the output. That's the token you should use here. diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index f753be58037..f2c89f76432 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -21,6 +21,7 @@ import ( ) // KubeadmConfig TODO add description +// TODO(phase1+) @krousey: Please don't embed structs. It obfuscates the source of the fields and doesn't really buy you anything. type KubeadmConfig struct { InitFlags JoinFlags @@ -35,6 +36,11 @@ type KubeadmConfig struct { // TODO(phase2) should we add validation functions for these structs? +// TODO(phase1+) refactor token handling +// - https://github.com/kubernetes/kubernetes/pull/33262/files#r80333662 +// - https://github.com/kubernetes/kubernetes/pull/33262/files#r80336374 +// - https://github.com/kubernetes/kubernetes/pull/33262/files#r80333982 + // InitFlags holds values for "kubeadm init" command flags. type InitFlags struct { API struct { @@ -63,7 +69,7 @@ type InitFlags struct { const ( DefaultServiceDNSDomain = "cluster.local" - DefaultServicesCIDRString = "100.64.0.0/12" + DefaultServicesCIDRString = "100.64.0.0/12" // Carrier-grade NAT range (RFC 6598) DefaultKubernetesVersion = "v1.4.0-beta.10" ) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index b4548bd202e..397db069ca0 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net" + "strings" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -43,10 +44,10 @@ var ( // NewCmdInit returns "kubeadm init" command. func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - advertiseAddrs := &[]string{} + advertiseAddrs := &[]string{} // TODO(pahse1+) make it work somehow else, custom flag or whatever cmd := &cobra.Command{ Use: "init", - Short: "Run this on the first machine.", + Short: "Run this in order to set up the Kubernetes master.", Run: func(cmd *cobra.Command, args []string) { err := RunInit(out, cmd, args, s, advertiseAddrs) cmdutil.CheckErr(err) @@ -55,58 +56,58 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd.PersistentFlags().StringVar( &s.Secrets.GivenToken, "token", "", - `(optional) Shared secret used to secure cluster bootstrap. If none is provided, one will be generated for you.`, + "Shared secret used to secure cluster bootstrap; if none is provided, one will be generated for you", ) cmd.PersistentFlags().StringSliceVar( advertiseAddrs, "api-advertise-addresses", []string{}, - `(optional) The IP addresses to advertise, in case autodetection fails.`, + "The IP addresses to advertise, in case autodetection fails", ) cmd.PersistentFlags().StringSliceVar( &s.InitFlags.API.ExternalDNSNames, "api-external-dns-names", []string{}, - `(optional) The DNS names to advertise, in case you have configured them yourself.`, + "The DNS names to advertise, in case you have configured them yourself", ) cmd.PersistentFlags().IPNetVar( &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, - `(optional) Use alterantive range of IP address for service VIPs, defaults to `+ + `Use alterantive range of IP address for service VIPs, defaults to `+ kubeadmapi.DefaultServicesCIDRString, ) cmd.PersistentFlags().IPNetVar( &s.InitFlags.PodNetwork.CIDR, "pod-network-cidr", net.IPNet{}, - `(optional) Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.`, + "Specify range of IP addresses for the pod network; if set, the control plane will automatically allocate CIDRs for every node", ) cmd.PersistentFlags().StringVar( &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, - `(optional) Use alternative domain for services, e.g. "myorg.internal"`, + `Use alternative domain for services, e.g. "myorg.internal"`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.CloudProvider, "cloud-provider", "", - `(optional) Enable a specific cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, + `Enable cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, ) cmd.PersistentFlags().BoolVar( &s.InitFlags.Schedulable, "schedule-pods-here", false, - `(optional) Allow to schedule workload to the node`, + `Allow to schedule workload to the node`, ) cmd.PersistentFlags().StringVar( &s.InitFlags.Versions.Kubernetes, "use-kubernetes-version", kubeadmapi.DefaultKubernetesVersion, - `(optional) Choose a specific Kubernetes version for the control plane`, + `Choose a specific Kubernetes version for the control plane`, ) // TODO (phase1+) @errordeveloper make the flags below not show up in --help but rather on --advanced-help cmd.PersistentFlags().StringSliceVar( &s.InitFlags.API.Etcd.ExternalEndpoints, "external-etcd-endpoints", []string{}, - `(optional) etcd endpoints to use, in case you have an external cluster.`, + "etcd endpoints to use, in case you have an external cluster", ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalCAFile, "external-etcd-cafile", "", - `(optional) etcd certificate authority certificate file."`, + "etcd certificate authority certificate file", ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalCertFile, "external-etcd-certfile", "", - `(optional) etcd client certificate file."`, + "etcd client certificate file", ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalKeyFile, "external-etcd-keyfile", "", - `(optional) etcd client key file."`, + "etcd client key file", ) return cmd @@ -126,14 +127,15 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub for _, i := range *advertiseAddrs { addr := net.ParseIP(i) if addr == nil { - return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addresses="+i) + // TODO(phase1+) custom flag will help to get this error message into a better place + return fmt.Errorf(" failed to parse %q (in %q) as an IP address", "--api-advertise-addresses="+strings.Join(*advertiseAddrs, ",")) } s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) } } + // TODO(phase1+) create a custom flag if s.InitFlags.CloudProvider != "" { - // TODO(phase2) we should be able to auto-detect it and check whether things like IAM roles are correct if _, ok := kubeadmapi.SupportedCloudProviders[s.InitFlags.CloudProvider]; !ok { return fmt.Errorf(" cloud provider %q is not supported, you can use any of %v, or leave it unset", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders) } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index a5fe58ab975..78c3b76a029 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -54,7 +54,7 @@ func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd.PersistentFlags().StringVar( &s.Secrets.GivenToken, "token", "", - `Shared secret used to secure bootstrap. Must match the output of 'kubeadm init'.`, + "(required) Shared secret used to secure bootstrap. Must match the output of 'kubeadm init'", ) return cmd @@ -77,7 +77,7 @@ func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub ok, err := kubeadmutil.UseGivenTokenIfValid(s) if !ok { if err != nil { - return fmt.Errorf(" %s (see --help)\n", err) + return fmt.Errorf(" %v (see --help)\n", err) } return fmt.Errorf("Must specify --token (see --help)\n") } diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index e95be511332..9e2759115c7 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -56,8 +56,8 @@ func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig, architecture string) ap // and accepts `--master` at the same time // // clever options include: - // - do CSR dance and create kubeconfig and mount it as secrete - // - create a service account with a second secret enconding kubeconfig + // - do CSR dance and create kubeconfig and mount it as a secret + // - create a service account with a second secret encoding kubeconfig // - use init container to convert known information to kubeconfig // - ...whatever Name: "kubeconfig", @@ -215,7 +215,7 @@ func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, error) { ip, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 10) if err != nil { - return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%s]", s.InitFlags.Services.CIDR, err) + return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%v]", s.InitFlags.Services.CIDR, err) } svc := &api.ServiceSpec{ @@ -238,7 +238,7 @@ func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Client SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { - return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + return fmt.Errorf(" failed creating essential kube-proxy addon [%v]", err) } } @@ -248,17 +248,17 @@ func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Client SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { - return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) + return fmt.Errorf(" failed creating essential kube-dns addon [%v]", err) } kubeDNSServiceSpec, err := createKubeDNSServiceSpec(s) if err != nil { - return fmt.Errorf(" failed creating essential kube-dns addon - %s", err) + return fmt.Errorf(" failed creating essential kube-dns addon - %v", err) } kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec) if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { - return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) + return fmt.Errorf(" failed creating essential kube-dns addon [%v]", err) } fmt.Println(" created essential addon: kube-dns") diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index ed6cd99c02f..c21d9a51c0d 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -39,14 +39,14 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli &clientcmd.ConfigOverrides{}, ).ClientConfig() if err != nil { - return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + return nil, fmt.Errorf(" failed to create API client configuration [%v]", err) } fmt.Println(" created API client configuration") client, err := clientset.NewForConfig(adminClientConfig) if err != nil { - return nil, fmt.Errorf(" failed to create API client [%s]", err) + return nil, fmt.Errorf(" failed to create API client [%v]", err) } fmt.Println(" created API client, waiting for the control plane to become ready") @@ -151,7 +151,7 @@ func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) * func findMyself(client *clientset.Clientset) (*api.Node, error) { nodeList, err := client.Nodes().List(api.ListOptions{}) if err != nil { - return nil, fmt.Errorf("unable to list nodes [%s]", err) + return nil, fmt.Errorf("unable to list nodes [%v]", err) } if len(nodeList.Items) < 1 { return nil, fmt.Errorf("no nodes found") @@ -187,9 +187,10 @@ func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, sched } func UpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, schedulable bool) error { + // TODO(phase1+) use iterate instead of recursion err := attemptToUpdateMasterRoleLabelsAndTaints(client, schedulable) if err != nil { - return fmt.Errorf(" failed to update master node - %s", err) + return fmt.Errorf(" failed to update master node - %v", err) } return nil } diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 54401ef1ae6..eb94f3e8c22 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -111,10 +111,10 @@ func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *cli kd := newKubeDiscovery(s, caCert) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { - return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoveryName, err) + return fmt.Errorf(" failed to create %q deployment [%v]", kubeDiscoveryName, err) } if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil { - return fmt.Errorf(" failed to create %q secret [%s]", kubeDiscoverySecretName, err) + return fmt.Errorf(" failed to create %q secret [%v]", kubeDiscoverySecretName, err) } fmt.Println(" created essential addon: kube-discovery, waiting for it to become ready") diff --git a/cmd/kubeadm/app/master/kubeconfig.go b/cmd/kubeadm/app/master/kubeconfig.go index 3ad4c3be554..d94c20dcb84 100644 --- a/cmd/kubeadm/app/master/kubeconfig.go +++ b/cmd/kubeadm/app/master/kubeconfig.go @@ -44,7 +44,7 @@ func CreateCertsAndConfigForClients(s *kubeadmapi.KubeadmConfig, clientNames []s for _, client := range clientNames { key, cert, err := newClientKeyAndCert(caCert, caKey) if err != nil { - return nil, fmt.Errorf(" failure while creating %s client certificate - %s", client, err) + return nil, fmt.Errorf(" failure while creating %s client certificate - %v", client, err) } config := kubeadmutil.MakeClientConfigWithCerts( basicClientConfig, diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 24cfeca6afb..0b99c6093e6 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -33,8 +33,7 @@ import ( "k8s.io/kubernetes/pkg/util/intstr" ) -// Static pod definitions in golang form are included below so that `kubeadm -// init master` and `kubeadm manual bootstrap master` can get going. +// Static pod definitions in golang form are included below so that `kubeadm init` can get going. const ( DefaultClusterName = "kubernetes" @@ -110,16 +109,16 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { manifestsPath := path.Join(s.EnvParams["kubernetes_dir"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) + return fmt.Errorf(" failed to create directory %q [%v]", manifestsPath, err) } for name, spec := range staticPodSpecs { filename := path.Join(manifestsPath, name+".json") serialized, err := json.MarshalIndent(spec, "", " ") if err != nil { - return fmt.Errorf(" failed to marshall manifest for %q to JSON [%s]", name, err) + return fmt.Errorf(" failed to marshall manifest for %q to JSON [%v]", name, err) } if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), filename); err != nil { - return fmt.Errorf(" failed to create static pod manifest file for %q (%q) [%s]", name, filename, err) + return fmt.Errorf(" failed to create static pod manifest file for %q (%q) [%v]", name, filename, err) } } return nil diff --git a/cmd/kubeadm/app/master/pki.go b/cmd/kubeadm/app/master/pki.go index 2cfed6d8b9e..c65b0eb0598 100644 --- a/cmd/kubeadm/app/master/pki.go +++ b/cmd/kubeadm/app/master/pki.go @@ -30,7 +30,7 @@ import ( func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { - return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + return nil, nil, fmt.Errorf("unable to create private key [%v]", err) } config := certutil.Config{ @@ -39,7 +39,7 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { cert, err := certutil.NewSelfSignedCACert(config, key) if err != nil { - return nil, nil, fmt.Errorf("unable to create self-singed certificate [%s]", err) + return nil, nil, fmt.Errorf("unable to create self-signed certificate [%v]", err) } return key, cert, nil @@ -48,7 +48,7 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { - return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) + return nil, nil, fmt.Errorf("unabel to create private key [%v]", err) } internalAPIServerFQDN := []string{ @@ -60,7 +60,7 @@ func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 1) if err != nil { - return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]", &s.InitFlags.Services.CIDR, err) + return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%v]", &s.InitFlags.Services.CIDR, err) } altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP) @@ -72,7 +72,7 @@ func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, } cert, err := certutil.NewSignedCert(config, key, caCert, caKey) if err != nil { - return nil, nil, fmt.Errorf("unable to sing certificate [%s]", err) + return nil, nil, fmt.Errorf("unable to sign certificate [%v]", err) } return key, cert, nil @@ -81,7 +81,7 @@ func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, func newClientKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { - return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + return nil, nil, fmt.Errorf("unable to create private key [%v]", err) } config := certutil.Config{ @@ -89,7 +89,7 @@ func newClientKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*rsa. } cert, err := certutil.NewSignedCert(config, key, caCert, caKey) if err != nil { - return nil, nil, fmt.Errorf("unable to sign certificate [%s]", err) + return nil, nil, fmt.Errorf("unable to sign certificate [%v]", err) } return key, cert, nil @@ -104,20 +104,20 @@ func writeKeysAndCert(pkiPath string, name string, key *rsa.PrivateKey, cert *x5 if key != nil { if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil { - return fmt.Errorf("unable to write private key file (%q) [%s]", privateKeyPath, err) + return fmt.Errorf("unable to write private key file (%q) [%v]", privateKeyPath, err) } if pubKey, err := certutil.EncodePublicKeyPEM(&key.PublicKey); err == nil { if err := certutil.WriteKey(publicKeyPath, pubKey); err != nil { - return fmt.Errorf("unable to write public key file (%q) [%s]", publicKeyPath, err) + return fmt.Errorf("unable to write public key file (%q) [%v]", publicKeyPath, err) } } else { - return fmt.Errorf("unable to encode public key to PEM [%s]", err) + return fmt.Errorf("unable to encode public key to PEM [%v]", err) } } if cert != nil { if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil { - return fmt.Errorf("unable to write certificate file (%q) [%s]", certificatePath, err) + return fmt.Errorf("unable to write certificate file (%q) [%v]", certificatePath, err) } } @@ -132,47 +132,46 @@ func newServiceAccountKey() (*rsa.PrivateKey, error) { return key, nil } +// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. +// It first generates a self-signed CA certificate, a server certificate (signed by the CA) and a key for +// signing service account tokens. It returns CA key and certificate, which is convenient for use with +// client config funcs. func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certificate, error) { var ( err error altNames certutil.AltNames ) - if len(s.InitFlags.API.AdvertiseAddrs) > 0 { - altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...) - } - - if len(s.InitFlags.API.ExternalDNSNames) > 0 { - altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSNames...) - } + altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...) + altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSNames...) pkiPath := path.Join(s.EnvParams["host_pki_path"]) caKey, caCert, err := newCertificateAuthority() if err != nil { - return nil, nil, fmt.Errorf(" failure while creating CA keys and certificate - %s", err) + return nil, nil, fmt.Errorf(" failure while creating CA keys and certificate - %v", err) } if err := writeKeysAndCert(pkiPath, "ca", caKey, caCert); err != nil { - return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %s", err) + return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %v", err) } apiKey, apiCert, err := newServerKeyAndCert(s, caCert, caKey, altNames) if err != nil { - return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %s", err) + return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %v", err) } if err := writeKeysAndCert(pkiPath, "apiserver", apiKey, apiCert); err != nil { - return nil, nil, fmt.Errorf(" failure while saving API server keys and certificate - %s", err) + return nil, nil, fmt.Errorf(" failure while saving API server keys and certificate - %v", err) } saKey, err := newServiceAccountKey() if err != nil { - return nil, nil, fmt.Errorf(" failure while creating service account signing keys [%s]", err) + return nil, nil, fmt.Errorf(" failure while creating service account signing keys [%v]", err) } if err := writeKeysAndCert(pkiPath, "sa", saKey, nil); err != nil { - return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) + return nil, nil, fmt.Errorf(" failure while saving service account signing keys - %v", err) } // TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certificates diff --git a/cmd/kubeadm/app/master/tokens.go b/cmd/kubeadm/app/master/tokens.go index 8f0cfd1c5ee..dd1b6f541da 100644 --- a/cmd/kubeadm/app/master/tokens.go +++ b/cmd/kubeadm/app/master/tokens.go @@ -30,6 +30,7 @@ import ( func generateTokenIfNeeded(s *kubeadmapi.KubeadmConfig) error { ok, err := kubeadmutil.UseGivenTokenIfValid(s) + // TODO(phase1+) @krousey: I know it won't happen with the way it is currently implemented, but this doesn't handle case where ok is true and err is non-nil. if !ok { if err != nil { return err @@ -49,14 +50,15 @@ func generateTokenIfNeeded(s *kubeadmapi.KubeadmConfig) error { func CreateTokenAuthFile(s *kubeadmapi.KubeadmConfig) error { tokenAuthFilePath := path.Join(s.EnvParams["host_pki_path"], "tokens.csv") if err := generateTokenIfNeeded(s); err != nil { - return fmt.Errorf(" failed to generate token(s) [%s]", err) + return fmt.Errorf(" failed to generate token(s) [%v]", err) } if err := os.MkdirAll(s.EnvParams["host_pki_path"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["host_pki_path"], err) + return fmt.Errorf(" failed to create directory %q [%v]", s.EnvParams["host_pki_path"], err) } serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", s.Secrets.BearerToken, uuid.NewUUID())) + // DumpReaderToFile create a file with mode 0600 if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil { - return fmt.Errorf(" failed to save token auth file (%q) [%s]", tokenAuthFilePath, err) + return fmt.Errorf(" failed to save token auth file (%q) [%v]", tokenAuthFilePath, err) } return nil } diff --git a/cmd/kubeadm/app/node/csr.go b/cmd/kubeadm/app/node/csr.go index 8bd61e12451..17c94283024 100644 --- a/cmd/kubeadm/app/node/csr.go +++ b/cmd/kubeadm/app/node/csr.go @@ -49,12 +49,12 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert &clientcmd.ConfigOverrides{}, ).ClientConfig() if err != nil { - return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + return nil, fmt.Errorf(" failed to create API client configuration [%v]", err) } client, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) if err != nil { - return nil, fmt.Errorf(" failed to create API client [%s]", err) + return nil, fmt.Errorf(" failed to create API client [%v]", err) } csrClient := client.CertificateSigningRequests() @@ -63,19 +63,19 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert // (for example user may be bringing up machines in parallel and for some reasons master is slow to boot) if err := checkCertsAPI(bootstrapClientConfig); err != nil { - return nil, fmt.Errorf(" fialed to proceed due to API compatibility issue - %s", err) + return nil, fmt.Errorf(" fialed to proceed due to API compatibility issue - %v", err) } fmt.Println(" created API client to obtain unique certificate for this node, generating keys and certificate signing request") key, err := certutil.MakeEllipticPrivateKeyPEM() if err != nil { - return nil, fmt.Errorf(" failed to generating private key [%s]", err) + return nil, fmt.Errorf(" failed to generating private key [%v]", err) } cert, err := csr.RequestNodeCertificate(csrClient, key, nodeName) if err != nil { - return nil, fmt.Errorf(" failed to request signed certificate from the API server [%s]", err) + return nil, fmt.Errorf(" failed to request signed certificate from the API server [%v]", err) } // TODO(phase1+) print some basic info about the cert @@ -93,13 +93,13 @@ func checkCertsAPI(config *restclient.Config) error { discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) if err != nil { - return fmt.Errorf("failed to create API discovery client [%s]", err) + return fmt.Errorf("failed to create API discovery client [%v]", err) } serverGroups, err := discoveryClient.ServerGroups() if err != nil { - return fmt.Errorf("failed to retrieve a list of supported API objects [%s]", err) + return fmt.Errorf("failed to retrieve a list of supported API objects [%v]", err) } for _, group := range serverGroups.Groups { @@ -110,7 +110,7 @@ func checkCertsAPI(config *restclient.Config) error { version, err := discoveryClient.ServerVersion() if err != nil { - return fmt.Errorf("unable to obtain API version [%s]", err) + return fmt.Errorf("unable to obtain API version [%v]", err) } return fmt.Errorf("API version %s does not support certificates API, use v1.4.0 or newer", version.String()) diff --git a/cmd/kubeadm/app/node/discovery.go b/cmd/kubeadm/app/node/discovery.go index 969557fca16..f5c82a63556 100644 --- a/cmd/kubeadm/app/node/discovery.go +++ b/cmd/kubeadm/app/node/discovery.go @@ -33,14 +33,14 @@ func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Conf requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, s.Secrets.TokenID) req, err := http.NewRequest("GET", requestURL, nil) if err != nil { - return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) + return nil, fmt.Errorf(" failed to consturct an HTTP request [%v]", err) } fmt.Printf(" created cluster info discovery client, requesting info from %q\n", requestURL) res, err := http.DefaultClient.Do(req) if err != nil { - return nil, fmt.Errorf(" failed to request cluster info [%s]", err) + return nil, fmt.Errorf(" failed to request cluster info [%v]", err) } buf := new(bytes.Buffer) io.Copy(buf, res.Body) @@ -48,20 +48,20 @@ func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Conf object, err := jose.ParseSigned(buf.String()) if err != nil { - return nil, fmt.Errorf(" failed to parse response as JWS object [%s]", err) + return nil, fmt.Errorf(" failed to parse response as JWS object [%v]", err) } fmt.Println(" cluster info object received, verifying signature using given token") output, err := object.Verify(s.Secrets.Token) if err != nil { - return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%v]", err) } clusterInfo := kubeadmapi.ClusterInfo{} if err := json.Unmarshal(output, &clusterInfo); err != nil { - return nil, fmt.Errorf(" failed to decode received cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to decode received cluster info object [%v]", err) } if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 { diff --git a/cmd/kubeadm/app/util/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig.go index 934e76c3dc7..40553b1c80c 100644 --- a/cmd/kubeadm/app/util/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig.go @@ -77,19 +77,19 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { if err := os.MkdirAll(s.EnvParams["kubernetes_dir"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["kubernetes_dir"], err) + return fmt.Errorf(" failed to create directory %q [%v]", s.EnvParams["kubernetes_dir"], err) } filename := path.Join(s.EnvParams["kubernetes_dir"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) if err != nil { - return fmt.Errorf(" failed to create %q, it already exists [%s]", filename, err) + return fmt.Errorf(" failed to create %q, it already exists [%v]", filename, err) } f.Close() if err := clientcmd.WriteToFile(*kubeconfig, filename); err != nil { - return fmt.Errorf(" failed to write to %q [%s]", filename, err) + return fmt.Errorf(" failed to write to %q [%v]", filename, err) } fmt.Printf(" created %q\n", filename) From b17e107defd17cb409ddd59fb013a652e9a35399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 25 Sep 2016 22:38:39 +0300 Subject: [PATCH 30/32] Various improvements plus added a version command --- cmd/kubeadm/app/api/types.go | 8 +----- cmd/kubeadm/app/cmd/cmd.go | 1 + cmd/kubeadm/app/cmd/init.go | 27 +++++++++++------- cmd/kubeadm/app/cmd/version.go | 44 +++++++++++++++++++++++++++++ cmd/kubeadm/app/kubeadm.go | 3 +- cmd/kubeadm/app/master/manifests.go | 6 ++-- hack/lib/golang.sh | 1 + 7 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/version.go diff --git a/cmd/kubeadm/app/api/types.go b/cmd/kubeadm/app/api/types.go index f2c89f76432..59999a11950 100644 --- a/cmd/kubeadm/app/api/types.go +++ b/cmd/kubeadm/app/api/types.go @@ -64,13 +64,12 @@ type InitFlags struct { Kubernetes string } CloudProvider string - Schedulable bool } const ( DefaultServiceDNSDomain = "cluster.local" DefaultServicesCIDRString = "100.64.0.0/12" // Carrier-grade NAT range (RFC 6598) - DefaultKubernetesVersion = "v1.4.0-beta.10" + DefaultKubernetesVersion = "v1.4.0" ) var ( @@ -86,15 +85,10 @@ var ( "rackspace", "vsphere", } - SupportedCloudProviders map[string]bool ) func init() { _, DefaultServicesCIDR, _ = net.ParseCIDR(DefaultServicesCIDRString) - SupportedCloudProviders = make(map[string]bool, len(ListOfCloudProviders)) - for _, v := range ListOfCloudProviders { - SupportedCloudProviders[v] = true - } } // JoinFlags holds values for "kubeadm join" command flags. diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index e46539719a9..7dd94c0a63b 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -84,6 +84,7 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env cmds.AddCommand(NewCmdInit(out, s)) cmds.AddCommand(NewCmdJoin(out, s)) + cmds.AddCommand(NewCmdVersion(out)) return cmds } diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 397db069ca0..ea259e52135 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -83,10 +83,6 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { &s.InitFlags.CloudProvider, "cloud-provider", "", `Enable cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, ) - cmd.PersistentFlags().BoolVar( - &s.InitFlags.Schedulable, "schedule-pods-here", false, - `Allow to schedule workload to the node`, - ) cmd.PersistentFlags().StringVar( &s.InitFlags.Versions.Kubernetes, "use-kubernetes-version", kubeadmapi.DefaultKubernetesVersion, `Choose a specific Kubernetes version for the control plane`, @@ -99,15 +95,15 @@ func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalCAFile, "external-etcd-cafile", "", - "etcd certificate authority certificate file", + "etcd certificate authority certificate file. Note: The path must be in /etc/ssl/certs", ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalCertFile, "external-etcd-certfile", "", - "etcd client certificate file", + "etcd client certificate file. Note: The path must be in /etc/ssl/certs", ) cmd.PersistentFlags().StringVar( &s.InitFlags.API.Etcd.ExternalKeyFile, "external-etcd-keyfile", "", - "etcd client key file", + "etcd client key file. Note: The path must be in /etc/ssl/certs", ) return cmd @@ -136,8 +132,18 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub // TODO(phase1+) create a custom flag if s.InitFlags.CloudProvider != "" { - if _, ok := kubeadmapi.SupportedCloudProviders[s.InitFlags.CloudProvider]; !ok { - return fmt.Errorf(" cloud provider %q is not supported, you can use any of %v, or leave it unset", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders) + found := false + for _, provider := range kubeadmapi.ListOfCloudProviders { + if provider == s.InitFlags.CloudProvider { + found = true + break + } + } + + if found { + fmt.Printf(" cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", s.InitFlags.CloudProvider) + } else { + return fmt.Errorf(" cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders) } } @@ -179,7 +185,8 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub return err } - if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, s.Schedulable); err != nil { + schedulePodsOnMaster := false + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, schedulePodsOnMaster); err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/version.go b/cmd/kubeadm/app/cmd/version.go new file mode 100644 index 00000000000..40114489c02 --- /dev/null +++ b/cmd/kubeadm/app/cmd/version.go @@ -0,0 +1,44 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/version" +) + +func NewCmdVersion(out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "version", + Short: "Print the version of kubeadm", + Run: func(cmd *cobra.Command, args []string) { + err := RunVersion(out, cmd) + cmdutil.CheckErr(err) + }, + } + return cmd +} + +func RunVersion(out io.Writer, cmd *cobra.Command) error { + fmt.Fprintf(out, "kubeadm version: %#v\n", version.Get()) + return nil +} diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 1a288e800ca..b275443cc69 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "strings" + "runtime" "github.com/renstrom/dedent" "github.com/spf13/pflag" @@ -46,7 +47,7 @@ func getEnvParams() map[string]string { "host_pki_path": "/etc/kubernetes/pki", "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", - "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), + "discovery_image": fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), "etcd_image": "", "component_loglevel": "--v=4", } diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 0b99c6093e6..df0b7cbeb2c 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -220,7 +220,7 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { baseFlags := map[string][]string{ apiServer: { - "--address=127.0.0.1", + "--insecure-bind-address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), @@ -233,7 +233,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--allow-privileged", }, controllerManager: { - // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world + "--address=127.0.0.1", "--leader-elect", "--master=127.0.0.1:8080", "--cluster-name=" + DefaultClusterName, @@ -244,7 +244,7 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", }, scheduler: { - // TODO(phase1+): consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world + "--address=127.0.0.1", "--leader-elect", "--master=127.0.0.1:8080", }, diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index c0b5f544511..752349848bc 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -35,6 +35,7 @@ kube::golang::server_targets() { cmd/kube-apiserver cmd/kube-controller-manager cmd/kubelet + cmd/kubeadm cmd/kubemark cmd/hyperkube plugin/cmd/kube-scheduler From 51573860fa6d1ecd6f60be38a7fe86682a295337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 25 Sep 2016 23:10:44 +0300 Subject: [PATCH 31/32] Update CHANGELOG and gofmt --- CHANGELOG.md | 2 +- cmd/kubeadm/app/kubeadm.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2e47a84a5b..81ff10b6018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -228,7 +228,7 @@ This is the first release tracked via the use of the [kubernetes/features](https - **Cluster Lifecycle** - [alpha] Ensure critical cluster infrastructure pods (Heapster, DNS, etc.) can schedule by evicting regular pods when necessary to make the critical pods schedule. ([docs](http://kubernetes.io/docs/admin/rescheduler/#guaranteed-scheduling-of-critical-add-on-pods)) ([kubernetes/features#62](https://github.com/kubernetes/features/issues/62)) - [alpha] Simplifies bootstrapping of TLS secured communication between the API server and kubelet. ([docs](http://kubernetes.io/docs/admin/master-node-communication/#kubelet-tls-bootstrap)) ([kubernetes/features#43](https://github.com/kubernetes/features/issues/43)) - - [alpha] `kubeadm` tool makes install much easier. ([docs](http://kubernetes.io/docs/kubeadm/)) ([kubernetes/features#11](https://github.com/kubernetes/features/issues/11)) + - [alpha] The `kubeadm` tool makes it much easier to bootstrap Kubernetes. ([docs](http://kubernetes.io/docs/getting-started-guides/kubeadm/)) ([kubernetes/features#11](https://github.com/kubernetes/features/issues/11)) - **Federation** - [alpha] Creating a `Federated Ingress` is as simple as submitting an `Ingress` config/manifest to the Federation API Server. Federation then creates a single global VIP to load balance the incoming L7 traffic across all the registered clusters no matter in what regions the clusters are. GCE L7 LoadBalancer is the only supported implementation in this release. ([kubernetes/features#82](https://github.com/kubernetes/features/issues/82)) - [alpha] Creating a `Namespace` in federation causes matching `Namespace`s to be created in all the clusters registered with that federation. ([docs](http://kubernetes.io/docs/user-guide/federation/namespaces)) ([kubernetes/features#69](https://github.com/kubernetes/features/issues/69)) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index b275443cc69..c1543bc186c 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -19,8 +19,8 @@ package app import ( "fmt" "os" - "strings" "runtime" + "strings" "github.com/renstrom/dedent" "github.com/spf13/pflag" From 3a4613d3b25dd1944a7f26b6d2037c9e82fcde7e Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Sun, 25 Sep 2016 21:57:50 +0100 Subject: [PATCH 32/32] fix https://k8s-gubernator.appspot.com/build/kubernetes-jenkins/pr-logs/pull/33262/kubernetes-pull-verify-all/15586/ --- cmd/kubeadm/app/cmd/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ea259e52135..b651e0473f2 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -124,7 +124,7 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.Kub addr := net.ParseIP(i) if addr == nil { // TODO(phase1+) custom flag will help to get this error message into a better place - return fmt.Errorf(" failed to parse %q (in %q) as an IP address", "--api-advertise-addresses="+strings.Join(*advertiseAddrs, ",")) + return fmt.Errorf(" failed to parse %q (in %q) as an IP address", i, "--api-advertise-addresses="+strings.Join(*advertiseAddrs, ",")) } s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) }