diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 75198bb5382..a3696e45ee6 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -73,6 +73,10 @@ type KubeletFlags struct { // NodeIP is IP address of the node. // If set, kubelet will use this IP address for the node. NodeIP string + + // DockershimRootDirectory is the path to the dockershim root directory. Defaults to + // /var/lib/dockershim if unset. Exposed for integration testing (e.g. in OpenShift). + DockershimRootDirectory string } // KubeletServer encapsulates all of the parameters necessary for starting up @@ -90,8 +94,9 @@ func NewKubeletServer() *KubeletServer { api.Scheme.Convert(versioned, &config, nil) return &KubeletServer{ KubeletFlags: KubeletFlags{ - KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), - RequireKubeConfig: false, + KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), + RequireKubeConfig: false, + DockershimRootDirectory: "/var/lib/dockershim", }, KubeletConfiguration: config, } @@ -129,6 +134,9 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node") + + fs.StringVar(&f.DockershimRootDirectory, "experimental-dockershim-root-directory", f.DockershimRootDirectory, "Path to the dockershim root directory.") + fs.MarkHidden("experimental-dockershim-root-directory") } // addFlags adds flags for a specific componentconfig.KubeletConfiguration to the specified FlagSet diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 27b544af919..43816055535 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -812,7 +812,7 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *componentconfig.Kubele if kubeDeps.OSInterface == nil { kubeDeps.OSInterface = kubecontainer.RealOS{} } - k, err := builder(kubeCfg, kubeDeps, standaloneMode, kubeFlags.HostnameOverride, kubeFlags.NodeIP) + k, err := builder(kubeCfg, kubeDeps, standaloneMode, kubeFlags.HostnameOverride, kubeFlags.NodeIP, kubeFlags.DockershimRootDirectory) if err != nil { return fmt.Errorf("failed to create kubelet: %v", err) } @@ -892,11 +892,11 @@ func startKubelet(k kubelet.KubeletBootstrap, podCfg *config.PodConfig, kubeCfg } } -func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (k kubelet.KubeletBootstrap, err error) { +func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool, hostnameOverride, nodeIP, dockershimRootDir string) (k kubelet.KubeletBootstrap, err error) { // TODO: block until all sources have delivered at least one update to the channel, or break the sync loop // up into "per source" synchronizations - k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, standaloneMode, hostnameOverride, nodeIP) + k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, standaloneMode, hostnameOverride, nodeIP, dockershimRootDir) if err != nil { return nil, err } @@ -936,7 +936,7 @@ func parseResourceList(m componentconfig.ConfigurationMap) (v1.ResourceList, err // RunDockershim only starts the dockershim in current process. This is only used for cri validate testing purpose // TODO(random-liu): Move this to a separate binary. -func RunDockershim(c *componentconfig.KubeletConfiguration) error { +func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir string) error { // Create docker client. dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, c.ImagePullProgressDeadline.Duration) @@ -978,7 +978,7 @@ func RunDockershim(c *componentconfig.KubeletConfiguration) error { } ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage, - streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler) + streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler, dockershimRootDir) if err != nil { return err } diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 1125e5b926a..c88a3f3ad3b 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -46,7 +46,7 @@ func main() { verflag.PrintAndExitIfRequested() if s.ExperimentalDockershim { - if err := app.RunDockershim(&s.KubeletConfiguration); err != nil { + if err := app.RunDockershim(&s.KubeletConfiguration, s.DockershimRootDirectory); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index a5824ed051e..959513bc699 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -242,6 +242,7 @@ experimental-bootstrap-token-auth experimental-check-node-capabilities-before-mount experimental-cri experimental-dockershim +experimental-dockershim-root-directory experimental-fail-swap-on experimental-kernel-memcg-notification experimental-keystone-ca-file diff --git a/pkg/kubelet/dockershim/docker_checkpoint.go b/pkg/kubelet/dockershim/docker_checkpoint.go index 713b349212f..6d0136c56fa 100644 --- a/pkg/kubelet/dockershim/docker_checkpoint.go +++ b/pkg/kubelet/dockershim/docker_checkpoint.go @@ -84,7 +84,7 @@ type PersistentCheckpointHandler struct { store CheckpointStore } -func NewPersistentCheckpointHandler() (CheckpointHandler, error) { +func NewPersistentCheckpointHandler(dockershimRootDir string) (CheckpointHandler, error) { fstore, err := NewFileStore(filepath.Join(dockershimRootDir, sandboxCheckpointDir)) if err != nil { return nil, err diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index bc88a95dffb..08558c5dc4c 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -60,9 +60,6 @@ const ( defaultSeccompProfile = "unconfined" - // dockershimRootDir is the root directory for dockershim - dockershimRootDir = "/var/lib/dockershim" - // Internal docker labels used to identify whether a container is a sandbox // or a regular container. // TODO: This is not backward compatible with older containers. We will @@ -150,9 +147,9 @@ var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPat // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process. func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, - pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandler dockertools.ExecHandler) (DockerService, error) { + pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandler dockertools.ExecHandler, dockershimRootDir string) (DockerService, error) { c := dockertools.NewInstrumentedDockerInterface(client) - checkpointHandler, err := NewPersistentCheckpointHandler() + checkpointHandler, err := NewPersistentCheckpointHandler(dockershimRootDir) if err != nil { return nil, err } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6ea11f7633a..b0c7b3e245f 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -187,7 +187,7 @@ type KubeletBootstrap interface { } // create and initialize a Kubelet instance -type KubeletBuilder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (KubeletBootstrap, error) +type KubeletBuilder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride, nodeIP, dockershimRootDir string) (KubeletBootstrap, error) // KubeletDeps is a bin for things we might consider "injected dependencies" -- objects constructed // at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping @@ -282,7 +282,7 @@ func getRuntimeAndImageServices(config *componentconfig.KubeletConfiguration) (i // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules. // No initialization of Kubelet and its modules should happen here. -func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (*Kubelet, error) { +func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride, nodeIP, dockershimRootDir string) (*Kubelet, error) { if kubeCfg.RootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", kubeCfg.RootDirectory) } @@ -553,7 +553,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub // Create and start the CRI shim running as a grpc server. streamingConfig := getStreamingConfig(kubeCfg, kubeDeps) ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage, - streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, dockerExecHandler) + streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, dockerExecHandler, dockershimRootDir) if err != nil { return nil, err }