Merge pull request #9193 from brendandburns/release-0.18

Release 0.18.1

cluster/gke/util.sh

@@ -116,7 +116,7 @@ function kube-up() {
   detect-project >&2
 
   # Make the specified network if we need to.
-  if ! gcloud compute networks describe "${NETWORK}" &>/dev/null; then
+  if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
     echo "Creating new network: ${NETWORK}" >&2
     gcloud compute networks create "${NETWORK}" --project="${PROJECT}" --range "${NETWORK_RANGE}"
   else
@@ -125,7 +125,7 @@ function kube-up() {
 
   # Allow SSH on all nodes in the network. This doesn't actually check whether
   # such a rule exists, only whether we've created this exact rule.
-  if ! gcloud compute firewall-rules describe "${FIREWALL_SSH}" &>/dev/null; then
+  if ! gcloud compute firewall-rules --project "${PROJECT}" describe "${FIREWALL_SSH}" &>/dev/null; then
     echo "Creating new firewall for SSH: ${FIREWALL_SSH}" >&2
     gcloud compute firewall-rules create "${FIREWALL_SSH}" \
       --allow="tcp:22" \

cluster/saltbase/salt/kubelet/default

@@ -58,10 +58,12 @@
   {% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%}
 {% endif -%}
 
-# Run containers under the root cgroup.
+# Run containers under the root cgroup and create a system container.
+{% set system_container = "" -%}
 {% set cgroup_root = "" -%}
 {% if grains['os_family'] == 'Debian' -%}
+  {% set system_container = "--system-container=/system" -%}
   {% set cgroup_root = "--cgroup_root=/" -%}
 {% endif -%}
 
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{cloud_provider}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{configure_cbr0}} {{cgroup_root}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{cloud_provider}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}}"

cmd/kubelet/app/server.go

@@ -108,6 +108,7 @@ type KubeletServer struct {
 	CgroupRoot                     string
 	ContainerRuntime               string
 	DockerDaemonContainer          string
+	SystemContainer                string
 	ConfigureCBR0                  bool
 	MaxPods                        int
 
@@ -170,6 +171,7 @@ func NewKubeletServer() *KubeletServer {
 		CgroupRoot:                  "",
 		ContainerRuntime:            "docker",
 		DockerDaemonContainer:       "/docker-daemon",
+		SystemContainer:             "",
 		ConfigureCBR0:               false,
 	}
 }
@@ -228,7 +230,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ResourceContainer, "resource-container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).")
 	fs.StringVar(&s.CgroupRoot, "cgroup_root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
 	fs.StringVar(&s.ContainerRuntime, "container_runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.")
-	fs.StringVar(&s.DockerDaemonContainer, "docker-daemon-container", s.DockerDaemonContainer, "Optional resource-only container in which to place the Docker Daemon. Empty for no container (Default: /docker-daemon).")
+	fs.StringVar(&s.SystemContainer, "system-container", s.SystemContainer, "Optional resource-only container in which to place all non-kernel processes that are not already in a container. Empty for no container. Rolling back the flag requires a reboot. (Default: \"\").")
 	fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
 	fs.IntVar(&s.MaxPods, "max-pods", 100, "Number of Pods that can run on this Kubelet.")
 
@@ -347,6 +349,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		ContainerRuntime:          s.ContainerRuntime,
 		Mounter:                   mounter,
 		DockerDaemonContainer:     s.DockerDaemonContainer,
+		SystemContainer:           s.SystemContainer,
 		ConfigureCBR0:             s.ConfigureCBR0,
 		MaxPods:                   s.MaxPods,
 	}
@@ -513,6 +516,7 @@ func SimpleKubelet(client *client.Client,
 		ContainerRuntime:          "docker",
 		Mounter:                   mount.New(),
 		DockerDaemonContainer:     "/docker-daemon",
+		SystemContainer:           "",
 		MaxPods:                   32,
 	}
 	return &kcfg
@@ -648,6 +652,7 @@ type KubeletConfig struct {
 	ContainerRuntime               string
 	Mounter                        mount.Interface
 	DockerDaemonContainer          string
+	SystemContainer                string
 	ConfigureCBR0                  bool
 	MaxPods                        int
 }
@@ -701,6 +706,7 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.ContainerRuntime,
 		kc.Mounter,
 		kc.DockerDaemonContainer,
+		kc.SystemContainer,
 		kc.ConfigureCBR0,
 		kc.MaxPods)
 

pkg/kubectl/cmd/exec.go

@@ -80,27 +80,29 @@ type execParams struct {
 	tty           bool
 }
 
-func extractPodAndContainer(cmd *cobra.Command, args []string, p *execParams) (podName string, containerName string, err error) {
+func extractPodAndContainer(cmd *cobra.Command, argsIn []string, p *execParams) (podName string, containerName string, args []string, err error) {
-	if len(p.podName) == 0 && len(args) == 0 {
+	if len(p.podName) == 0 && len(argsIn) == 0 {
-		return "", "", cmdutil.UsageError(cmd, "POD is required for exec")
+		return "", "", nil, cmdutil.UsageError(cmd, "POD is required for exec")
 	}
 	if len(p.podName) != 0 {
 		printDeprecationWarning("exec POD", "-p POD")
 		podName = p.podName
-		if len(args) < 1 {
+		if len(argsIn) < 1 {
-			return "", "", cmdutil.UsageError(cmd, "COMMAND is required for exec")
+			return "", "", nil, cmdutil.UsageError(cmd, "COMMAND is required for exec")
 		}
+		args = argsIn
 	} else {
-		podName = args[0]
+		podName = argsIn[0]
-		if len(args) < 2 {
+		args = argsIn[1:]
-			return "", "", cmdutil.UsageError(cmd, "COMMAND is required for exec")
+		if len(args) < 1 {
+			return "", "", nil, cmdutil.UsageError(cmd, "COMMAND is required for exec")
 		}
 	}
-	return podName, p.containerName, nil
+	return podName, p.containerName, args, nil
 }
 
-func RunExec(f *cmdutil.Factory, cmd *cobra.Command, cmdIn io.Reader, cmdOut, cmdErr io.Writer, p *execParams, args []string, re remoteExecutor) error {
+func RunExec(f *cmdutil.Factory, cmd *cobra.Command, cmdIn io.Reader, cmdOut, cmdErr io.Writer, p *execParams, argsIn []string, re remoteExecutor) error {
-	podName, containerName, err := extractPodAndContainer(cmd, args, p)
+	podName, containerName, args, err := extractPodAndContainer(cmd, argsIn, p)
 	namespace, err := f.DefaultNamespace()
 	if err != nil {
 		return err

pkg/kubectl/cmd/exec_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"reflect"
 	"testing"
 
 	"github.com/spf13/cobra"
@@ -43,58 +44,73 @@ func TestPodAndContainer(t *testing.T) {
 	tests := []struct {
 		args              []string
 		p                 *execParams
+		name              string
 		expectError       bool
 		expectedPod       string
 		expectedContainer string
+		expectedArgs      []string
 	}{
 		{
 			p:           &execParams{},
 			expectError: true,
+			name:        "empty",
 		},
 		{
 			p:           &execParams{podName: "foo"},
 			expectError: true,
+			name:        "no cmd",
 		},
 		{
 			p:           &execParams{podName: "foo", containerName: "bar"},
 			expectError: true,
+			name:        "no cmd, w/ container",
 		},
 		{
 			p:            &execParams{podName: "foo"},
 			args:         []string{"cmd"},
 			expectedPod:  "foo",
+			expectedArgs: []string{"cmd"},
+			name:         "pod in flags",
 		},
 		{
 			p:           &execParams{},
 			args:        []string{"foo"},
 			expectError: true,
+			name:        "no cmd, w/o flags",
 		},
 		{
 			p:            &execParams{},
 			args:         []string{"foo", "cmd"},
 			expectedPod:  "foo",
+			expectedArgs: []string{"cmd"},
+			name:         "cmd, w/o flags",
 		},
 		{
 			p:                 &execParams{containerName: "bar"},
 			args:              []string{"foo", "cmd"},
 			expectedPod:       "foo",
 			expectedContainer: "bar",
+			expectedArgs:      []string{"cmd"},
+			name:              "cmd, container in flag",
 		},
 	}
 	for _, test := range tests {
 		cmd := &cobra.Command{}
-		podName, containerName, err := extractPodAndContainer(cmd, test.args, test.p)
+		podName, containerName, args, err := extractPodAndContainer(cmd, test.args, test.p)
 		if podName != test.expectedPod {
-			t.Errorf("expected: %s, got: %s", test.expectedPod, podName)
+			t.Errorf("expected: %s, got: %s (%s)", test.expectedPod, podName, test.name)
 		}
 		if containerName != test.expectedContainer {
-			t.Errorf("expected: %s, got: %s", test.expectedContainer, containerName)
+			t.Errorf("expected: %s, got: %s (%s)", test.expectedContainer, containerName, test.name)
 		}
 		if test.expectError && err == nil {
-			t.Error("unexpected non-error")
+			t.Errorf("unexpected non-error (%s)", test.name)
 		}
 		if !test.expectError && err != nil {
-			t.Errorf("unexpected error: %v", err)
+			t.Errorf("unexpected error: %v (%s)", err, test.name)
+		}
+		if !reflect.DeepEqual(test.expectedArgs, args) {
+			t.Errorf("expected: %v, got %v (%s)", test.expectedArgs, args, test.name)
 		}
 	}
 }

pkg/kubectl/run.go

@@ -29,6 +29,7 @@ type BasicReplicationController struct{}
 func (BasicReplicationController) ParamNames() []GeneratorParam {
 	return []GeneratorParam{
 		{"labels", false},
+		{"default-name", false},
 		{"name", true},
 		{"replicas", true},
 		{"image", true},

pkg/kubelet/container_manager.go

@@ -20,5 +20,6 @@ package kubelet
 type containerManager interface {
 	// Runs the container manager's housekeeping.
 	// - Ensures that the Docker daemon is in a container.
+	// - Creates the system container where all non-containerized processes run.
 	Start() error
 }

pkg/kubelet/container_manager_linux.go

@@ -35,33 +35,60 @@ import (
 )
 
 type containerManagerImpl struct {
-	// Absolute name of the desired container that Docker should be in.
+	// Whether to create and use the specified containers.
-	dockerContainerName string
+	useDockerContainer bool
+	useSystemContainer bool
 
-	// The manager of the resource-only container Docker should be in.
+	// OOM score for the Docker container.
-	manager           fs.Manager
 	dockerOomScoreAdj int
+
+	// Managers for containers.
+	dockerContainer fs.Manager
+	systemContainer fs.Manager
+	rootContainer   fs.Manager
 }
 
 var _ containerManager = &containerManagerImpl{}
 
-// Takes the absolute name that the Docker daemon should be in.
+// Takes the absolute name of the specified containers.
-// Empty container name disables moving the Docker daemon.
+// Empty container name disables use of the specified container.
-func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+func newContainerManager(dockerDaemonContainer, systemContainer string) (containerManager, error) {
+	if systemContainer == "/" {
+		return nil, fmt.Errorf("system container cannot be root (\"/\")")
+	}
+
 	return &containerManagerImpl{
-		dockerContainerName: dockerDaemonContainer,
+		useDockerContainer: dockerDaemonContainer != "",
-		manager: fs.Manager{
+		useSystemContainer: systemContainer != "",
+		dockerOomScoreAdj:  -900,
+		dockerContainer: fs.Manager{
 			Cgroups: &configs.Cgroup{
 				Name:            dockerDaemonContainer,
 				AllowAllDevices: true,
 			},
 		},
-		dockerOomScoreAdj: -900,
+		systemContainer: fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name:            systemContainer,
+				AllowAllDevices: true,
+			},
+		},
+		rootContainer: fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name: "/",
+			},
+		},
 	}, nil
 }
 
 func (cm *containerManagerImpl) Start() error {
-	if cm.dockerContainerName != "" {
+	if cm.useSystemContainer {
+		err := cm.ensureSystemContainer()
+		if err != nil {
+			return err
+		}
+	}
+	if cm.useDockerContainer {
 		go util.Until(func() {
 			err := cm.ensureDockerInContainer()
 			if err != nil {
@@ -99,10 +126,10 @@ func (cm *containerManagerImpl) ensureDockerInContainer() error {
 			errs = append(errs, fmt.Errorf("failed to find container of PID %q: %v", pid, err))
 		}
 
-		if cont != cm.dockerContainerName {
+		if cont != cm.dockerContainer.Cgroups.Name {
-			err = cm.manager.Apply(pid)
+			err = cm.dockerContainer.Apply(pid)
 			if err != nil {
-				errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainerName))
+				errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainer.Cgroups.Name))
 			}
 		}
 
@@ -125,3 +152,60 @@ func getContainer(pid int) (string, error) {
 
 	return cgroups.ParseCgroupFile("cpu", f)
 }
+
+// Ensures the system container is created and all non-kernel processes without
+// a container are moved to it.
+func (cm *containerManagerImpl) ensureSystemContainer() error {
+	// Move non-kernel PIDs to the system container.
+	attemptsRemaining := 10
+	var errs []error
+	for attemptsRemaining >= 0 {
+		// Only keep errors on latest attempt.
+		errs = []error{}
+		attemptsRemaining--
+
+		allPids, err := cm.rootContainer.GetPids()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("Failed to list PIDs for root: %v", err))
+			continue
+		}
+
+		// Remove kernel pids
+		pids := make([]int, 0, len(allPids))
+		for _, pid := range allPids {
+			if isKernelPid(pid) {
+				continue
+			}
+
+			pids = append(pids, pid)
+		}
+		glog.Infof("Found %d PIDs in root, %d of them are kernel related", len(allPids), len(allPids)-len(pids))
+
+		// Check if we moved all the non-kernel PIDs.
+		if len(pids) == 0 {
+			break
+		}
+
+		glog.Infof("Moving non-kernel threads: %v", pids)
+		for _, pid := range pids {
+			err := cm.systemContainer.Apply(pid)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to move PID %d into the system container %q: %v", pid, cm.systemContainer.Cgroups.Name, err))
+				continue
+			}
+		}
+
+	}
+	if attemptsRemaining < 0 {
+		errs = append(errs, fmt.Errorf("ran out of attempts to create system containers %q", cm.systemContainer.Cgroups.Name))
+	}
+
+	return errors.NewAggregate(errs)
+}
+
+// Determines whether the specified PID is a kernel PID.
+func isKernelPid(pid int) bool {
+	// Kernel threads have no associated executable.
+	_, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
+	return err != nil
+}

pkg/kubelet/container_manager_unsupported.go

@@ -31,6 +31,6 @@ func (unsupportedContainerManager) Start() error {
 	return fmt.Errorf("Container Manager is unsupported in this build")
 }
 
-func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+func newContainerManager(dockerDaemonContainer, systemContainer string) (containerManager, error) {
 	return &unsupportedContainerManager{}, nil
 }

pkg/kubelet/kubelet.go

@@ -139,6 +139,7 @@ func NewMainKubelet(
 	containerRuntime string,
 	mounter mount.Interface,
 	dockerDaemonContainer string,
+	systemContainer string,
 	configureCBR0 bool,
 	pods int) (*Kubelet, error) {
 	if rootDirectory == "" {
@@ -147,6 +148,9 @@ func NewMainKubelet(
 	if resyncInterval <= 0 {
 		return nil, fmt.Errorf("invalid sync frequency %d", resyncInterval)
 	}
+	if systemContainer != "" && cgroupRoot == "" {
+		return nil, fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
+	}
 	dockerClient = dockertools.NewInstrumentedDockerInterface(dockerClient)
 
 	serviceStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
@@ -295,7 +299,9 @@ func NewMainKubelet(
 		return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime)
 	}
 
-	containerManager, err := newContainerManager(dockerDaemonContainer)
+	// Setup container manager, can fail if the devices hierarchy is not mounted
+	// (it is required by Docker however).
+	containerManager, err := newContainerManager(dockerDaemonContainer, systemContainer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create the Container Manager: %v", err)
 	}

pkg/version/base.go

@@ -36,8 +36,8 @@ package version
 var (
 	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead.
 	gitMajor     string = "0"              // major version, always numeric
-	gitMinor     string = "18.0"           // minor version, numeric possibly followed by "+"
+	gitMinor     string = "18.1+"          // minor version, numeric possibly followed by "+"
-	gitVersion   string = "v0.18.0"        // version from git, output of $(git describe)
+	gitVersion   string = "v0.18.1-dev"    // version from git, output of $(git describe)
 	gitCommit    string = ""               // sha1 from git, output of $(git rev-parse HEAD)
 	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"
 )