From d814d973ff48e6ff53b4107037032f41dedc5aba Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 18 Mar 2016 17:22:11 -0700 Subject: [PATCH] rkt: Bump rkt required version. Get options from api service. --- cmd/kubelet/app/options/options.go | 8 +++- cmd/kubelet/app/server.go | 3 ++ hack/verify-flags/known-flags.txt | 1 + pkg/apis/componentconfig/types.go | 10 +++-- pkg/kubelet/kubelet.go | 2 + pkg/kubelet/rkt/config.go | 60 +++++++++++++++++++++++++++--- pkg/kubelet/rkt/rkt.go | 32 +++++++++------- pkg/kubelet/rkt/rkt_test.go | 1 - 8 files changed, 92 insertions(+), 25 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 848348af10b..431e7dcfd60 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/kubelet/qos" + "k8s.io/kubernetes/pkg/kubelet/rkt" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/util" @@ -114,6 +115,7 @@ func NewKubeletServer() *KubeletServer { RegistryPullQPS: 5.0, KubeletCgroups: "", RktPath: "", + RktAPIEndpoint: rkt.DefaultRktAPIServiceEndpoint, RktStage1Image: "", RootDirectory: defaultRootDir, RuntimeCgroups: "", @@ -206,8 +208,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.CgroupRoot, "cgroup-root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.") fs.StringVar(&s.ContainerRuntime, "container-runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.") fs.StringVar(&s.LockFilePath, "lock-file", s.LockFilePath, " The path to file for kubelet to use as a lock file.") - fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH. Only used if --container-runtime='rkt'") - fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used") + fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH. Only used if --container-runtime='rkt'.") + fs.StringVar(&s.RktAPIEndpoint, "rkt-api-endpoint", s.RktAPIEndpoint, "The endpoint of the rkt API service to communicate with. Only used if --container-runtime='rkt'.") + fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used.") + fs.MarkDeprecated("rkt-stage1-image", "Will be removed in a future version. The default stage1 image will be specified by the rkt configurations, see https://github.com/coreos/rkt/blob/master/Documentation/configuration.md for more details.") fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.") fs.StringVar(&s.HairpinMode, "hairpin-mode", s.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".") fs.BoolVar(&s.BabysitDaemons, "babysit-daemons", s.BabysitDaemons, "If true, the node has babysitter process monitoring docker and kubelet.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index bae51fdb9b1..6ba0ab12ebd 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -243,6 +243,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { Reservation: *reservation, KubeletCgroups: s.KubeletCgroups, RktPath: s.RktPath, + RktAPIEndpoint: s.RktAPIEndpoint, RktStage1Image: s.RktStage1Image, RootDirectory: s.RootDirectory, Runonce: s.RunOnce, @@ -757,6 +758,7 @@ type KubeletConfig struct { ResolverConfig string KubeletCgroups string RktPath string + RktAPIEndpoint string RktStage1Image string RootDirectory string Runonce bool @@ -839,6 +841,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.CgroupRoot, kc.ContainerRuntime, kc.RktPath, + kc.RktAPIEndpoint, kc.RktStage1Image, kc.Mounter, kc.Writer, diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 997cf40f27d..bd360f2efd4 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -339,6 +339,7 @@ resource-container resource-quota-sync-period resource-version retry_time +rkt-api-endpoint rkt-path rkt-stage1-image root-ca-file diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index fa52286da2b..3234489ab28 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -257,16 +257,18 @@ type KubeletConfiguration struct { CgroupRoot string `json:"cgroupRoot,omitempty"` // containerRuntime is the container runtime to use. ContainerRuntime string `json:"containerRuntime"` - // rktPath is hte path of rkt binary. Leave empty to use the first rkt in + // rktPath is the path of rkt binary. Leave empty to use the first rkt in // $PATH. RktPath string `json:"rktPath,omitempty"` + // rktApiEndpoint is the endpoint of the rkt API service to communicate with. + RktAPIEndpoint string `json:"rktAPIEndpoint,omitempty"` + // rktStage1Image is the image to use as stage1. Local paths and + // http/https URLs are supported. + RktStage1Image string `json:"rktStage1Image,omitempty"` // lockFilePath is the path that kubelet will use to as a lock file. // It uses this file as a lock to synchronize with other kubelet processes // that may be running. LockFilePath string `json:"lockFilePath"` - // rktStage1Image is the image to use as stage1. Local paths and - // http/https URLs are supported. - RktStage1Image string `json:"rktStage1Image,omitempty"` // configureCBR0 enables the kublet to configure cbr0 based on // Node.Spec.PodCIDR. ConfigureCBR0 bool `json:"configureCbr0"` diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2af0f40e649..925b5ac9821 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -191,6 +191,7 @@ func NewMainKubelet( cgroupRoot string, containerRuntime string, rktPath string, + rktAPIEndpoint string, rktStage1Image string, mounter mount.Interface, writer kubeio.Writer, @@ -416,6 +417,7 @@ func NewMainKubelet( InsecureOptions: "image,ondisk", } rktRuntime, err := rkt.New( + rktAPIEndpoint, conf, klet, recorder, diff --git a/pkg/kubelet/rkt/config.go b/pkg/kubelet/rkt/config.go index 8b62b59ffcd..809eefc5431 100644 --- a/pkg/kubelet/rkt/config.go +++ b/pkg/kubelet/rkt/config.go @@ -16,7 +16,12 @@ limitations under the License. package rkt -import "fmt" +import ( + "fmt" + + rktapi "github.com/coreos/rkt/api/v1alpha" + "golang.org/x/net/context" +) // Config stores the global configuration for the rkt runtime. // Detailed documents can be found at: @@ -24,17 +29,21 @@ import "fmt" type Config struct { // The absolute path to the binary, or leave empty to find it in $PATH. Path string + // The rkt data directory. + Dir string // The image to use as stage1. Stage1Image string // The debug flag for rkt. Debug bool - // The rkt data directory. - Dir string // Comma-separated list of security features to disable. // Allowed values: "none", "image", "tls", "ondisk", "http", "all". InsecureOptions string // The local config directory. LocalConfigDir string + // The user config directory. + UserConfigDir string + // The system config directory. + SystemConfigDir string } // buildGlobalOptions returns an array of global command line options. @@ -44,13 +53,54 @@ func (c *Config) buildGlobalOptions() []string { return result } - result = append(result, fmt.Sprintf("--debug=%v", c.Debug)) - result = append(result, fmt.Sprintf("--insecure-options=%s", c.InsecureOptions)) + if c.Debug { + result = append(result, "--debug=true") + } + if c.InsecureOptions != "" { + result = append(result, fmt.Sprintf("--insecure-options=%s", c.InsecureOptions)) + } if c.LocalConfigDir != "" { result = append(result, fmt.Sprintf("--local-config=%s", c.LocalConfigDir)) } + if c.UserConfigDir != "" { + result = append(result, fmt.Sprintf("--user-config=%s", c.UserConfigDir)) + } + if c.SystemConfigDir != "" { + result = append(result, fmt.Sprintf("--system-config=%s", c.SystemConfigDir)) + } if c.Dir != "" { result = append(result, fmt.Sprintf("--dir=%s", c.Dir)) } return result } + +// getConfig gets configurations from the rkt API service +// and merge it with the existing config. The merge rule is +// that the fields in the provided config will override the +// result that get from the rkt api service. +func (r *Runtime) getConfig(cfg *Config) (*Config, error) { + resp, err := r.apisvc.GetInfo(context.Background(), &rktapi.GetInfoRequest{}) + if err != nil { + return nil, err + } + + flags := resp.Info.GlobalFlags + + if cfg.Dir == "" { + cfg.Dir = flags.Dir + } + if cfg.InsecureOptions == "" { + cfg.InsecureOptions = flags.InsecureFlags + } + if cfg.LocalConfigDir == "" { + cfg.LocalConfigDir = flags.LocalConfigDir + } + if cfg.UserConfigDir == "" { + cfg.UserConfigDir = flags.UserConfigDir + } + if cfg.SystemConfigDir == "" { + cfg.SystemConfigDir = flags.SystemConfigDir + } + + return cfg, nil +} diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 7d28b85ae9c..e8d3a4eafc4 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -52,11 +52,12 @@ import ( ) const ( - RktType = "rkt" + RktType = "rkt" + DefaultRktAPIServiceEndpoint = "localhost:15441" minimumAppcVersion = "0.7.4" - minimumRktBinVersion = "0.13.0" - recommendedRktBinVersion = "0.13.0" + minimumRktBinVersion = "1.2.1" + recommendedRktBinVersion = "1.2.1" minimumRktApiVersion = "1.0.0-alpha" minimumSystemdVersion = "219" @@ -108,9 +109,7 @@ type Runtime struct { // The grpc client for rkt api-service. apisvcConn *grpc.ClientConn apisvc rktapi.PublicAPIClient - // The absolute path to rkt binary. - rktBinAbsPath string - config *Config + config *Config // TODO(yifan): Refactor this to be generic keyring. dockerKeyring credentialprovider.DockerKeyring @@ -134,7 +133,9 @@ type volumeGetter interface { // New creates the rkt container runtime which implements the container runtime interface. // It will test if the rkt binary is in the $PATH, and whether we can get the // version of it. If so, creates the rkt container runtime, otherwise returns an error. -func New(config *Config, +func New( + apiEndpoint string, + config *Config, runtimeHelper kubecontainer.RuntimeHelper, recorder record.EventRecorder, containerRefManager *kubecontainer.RefManager, @@ -150,16 +151,16 @@ func New(config *Config, } // TODO(yifan): Use secure connection. - apisvcConn, err := grpc.Dial(defaultRktAPIServiceAddr, grpc.WithInsecure()) + apisvcConn, err := grpc.Dial(apiEndpoint, grpc.WithInsecure()) if err != nil { return nil, fmt.Errorf("rkt: cannot connect to rkt api service: %v", err) } - rktBinAbsPath := config.Path - if rktBinAbsPath == "" { + // TODO(yifan): Get the rkt path from API service. + if config.Path == "" { // No default rkt path was set, so try to find one in $PATH. var err error - rktBinAbsPath, err = exec.LookPath("rkt") + config.Path, err = exec.LookPath("rkt") if err != nil { return nil, fmt.Errorf("cannot find rkt binary: %v", err) } @@ -167,7 +168,6 @@ func New(config *Config, rkt := &Runtime{ systemd: systemd, - rktBinAbsPath: rktBinAbsPath, apisvcConn: apisvcConn, apisvc: rktapi.NewPublicAPIClient(apisvcConn), config: config, @@ -178,6 +178,12 @@ func New(config *Config, livenessManager: livenessManager, volumeGetter: volumeGetter, } + + rkt.config, err = rkt.getConfig(rkt.config) + if err != nil { + return nil, fmt.Errorf("rkt: cannot get config from rkt api service: %v", err) + } + if serializeImagePulls { rkt.imagePuller = kubecontainer.NewSerializedImagePuller(recorder, rkt, imageBackOff) } else { @@ -192,7 +198,7 @@ func New(config *Config, } func (r *Runtime) buildCommand(args ...string) *exec.Cmd { - cmd := exec.Command(r.rktBinAbsPath) + cmd := exec.Command(r.config.Path) cmd.Args = append(cmd.Args, r.config.buildGlobalOptions()...) cmd.Args = append(cmd.Args, args...) return cmd diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index e34930b8a5a..dd8ed73e0d5 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -1109,7 +1109,6 @@ func TestGenerateRunCommand(t *testing.T) { } rkt := &Runtime{ - rktBinAbsPath: "/bin/rkt/rkt", config: &Config{ Path: "/bin/rkt/rkt", Stage1Image: "/bin/rkt/stage1-coreos.aci",