Merge pull request #54935 from anguslees/kubeadm-chroot

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: chroot to new --rootfs arg

**What this PR does / why we need it**:

This change adds a new --rootfs=path option to kubeadm, and (if
provided) chroot()s to this path before performing file operations.

This makes it possible to run the kubeadm binary from a container, but
perform remaining file operations against the host filesystem using
something like:

```
    docker run -v /:/rootfs --net=host --uts=host --pid=host \
       kubeadm:latest init ...
```

(Assuming something like the included `examples/kubeadm/Dockerfile` which sets CMD to `kubeadm --rootfs=/rootfs` - Edit: Dockerfile has been removed from this PR, but you get the idea)

Fixes kubernetes/kubeadm#503

**Special notes for your reviewer**:

- I'm not sure where is best to put the Dockerfile, or hook it up to the build process.  Advice sought.

- The kubeadm command line arg handling was less unified than I was expecting to find.  I've implemented this arg for `init` and `join`.   I can add it to all the others too, if we're happy with the approach. An alternative would be to add the arg in the parent `KubeadmCommand`, possibly with a `PersistantFlag` - then it would automatically exist for all kubeadm subcommands.

- It would be slightly preferable if we could order `--rootfs` _before_ the subcommand so we could apply the arg automatically with `ENTRYPOINT ["kubeadm", "--rootfs=/rootfs"]`.  This would be the only such flag in `kubeadm` however, so I have not implemented it that way atm.  (Another alternative would be an env var)

**Release note**:
```release-note
Adds a new EXPERIMENTAL `--rootfs` flag to kubeadm, which (if specified) causes kubeadm to chroot before performing any file operations.  This is expected to be useful when setting up kubernetes on a different filesystem, such as invoking kubeadm from docker.
```

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -376,7 +376,7 @@ func ValidateMixedArguments(flag *pflag.FlagSet) error {
 
 	mixedInvalidFlags := []string{}
 	flag.Visit(func(f *pflag.Flag) {
-		if f.Name == "config" || f.Name == "ignore-preflight-errors" || strings.HasPrefix(f.Name, "skip-") || f.Name == "dry-run" || f.Name == "kubeconfig" || f.Name == "v" {
+		if f.Name == "config" || f.Name == "ignore-preflight-errors" || strings.HasPrefix(f.Name, "skip-") || f.Name == "dry-run" || f.Name == "kubeconfig" || f.Name == "v" || f.Name == "rootfs" {
 			// "--skip-*" flags or other whitelisted flags can be set with --config
 			return
 		}

cmd/kubeadm/app/cmd/cmd.go

@@ -21,15 +21,19 @@ import (
 
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/upgrade"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	// Register the kubeadm configuration types because CLI flag generation
 	// depends on the generated defaults.
 )
 
 // NewKubeadmCommand returns cobra.Command to run kubeadm command
 func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command {
+	var rootfsPath string
+
 	cmds := &cobra.Command{
 		Use:   "kubeadm",
 		Short: "kubeadm: easily bootstrap a secure Kubernetes cluster",
@@ -65,6 +69,15 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command {
 			    You can then repeat the second step on as many other machines as you like.
 
 		`),
+
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if rootfsPath != "" {
+				if err := kubeadmutil.Chroot(rootfsPath); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
 	}
 
 	cmds.ResetFlags()
@@ -86,5 +99,15 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command {
 	experimentalCmd.AddCommand(phases.NewCmdPhase(out))
 	cmds.AddCommand(experimentalCmd)
 
+	AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath)
+
 	return cmds
 }
+
+// AddKubeadmOtherFlags adds flags that are not bound to a configuration file to the given flagset
+func AddKubeadmOtherFlags(flagSet *pflag.FlagSet, rootfsPath *string) {
+	flagSet.StringVar(
+		rootfsPath, "rootfs", *rootfsPath,
+		"[EXPERIMENTAL] The path to the 'real' host root filesystem.",
+	)
+}

cmd/kubeadm/app/util/BUILD

@@ -11,6 +11,8 @@ go_library(
     srcs = [
         "arguments.go",
         "cgroupdriver.go",
+        "chroot_unix.go",
+        "chroot_windows.go",
         "copy.go",
         "endpoint.go",
         "error.go",
@@ -40,6 +42,7 @@ go_test(
     srcs = [
         "arguments_test.go",
         "cgroupdriver_test.go",
+        "chroot_test.go",
         "endpoint_test.go",
         "error_test.go",
         "marshal_test.go",

cmd/kubeadm/app/util/chroot_test.go

@@ -0,0 +1,66 @@
+// +build !windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Can't just call Chroot() because it will affect other tests in the
+// same process.  Golang makes it hard to just call fork(), so instead
+// we exec ourselves again, and run the test in the new subprocess.
+
+func testChrootReal(t *testing.T) {
+	testfile := filepath.FromSlash("/" + filepath.Base(os.Args[0]))
+	dir := filepath.Dir(os.Args[0])
+	if dir == "." {
+		t.Skip("skipping: running test at root somehow")
+	}
+	if err := Chroot(dir); err != nil {
+		if strings.Contains(err.Error(), "operation not permitted") {
+			t.Skip("skipping: insufficient permissions to chroot")
+		}
+		t.Fatalf("chroot error: %v", err)
+	}
+
+	// All file access should now be relative to `dir`
+	if _, err := os.Stat(testfile); err != nil {
+		t.Errorf("Expected file %q to exist, but got %v", testfile, err)
+	}
+}
+
+func TestChroot(t *testing.T) {
+	if os.Getenv("GO_TEST_CHROOT_FOR_REALZ") == "1" {
+		testChrootReal(t)
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.v", "-test.run=TestChroot")
+	cmd.Env = []string{"GO_TEST_CHROOT_FOR_REALZ=1"}
+
+	out, err := cmd.Output()
+	t.Logf("subprocess output:\n%s", out)
+	if err != nil {
+		t.Errorf("subprocess error: %v", err)
+	}
+}

cmd/kubeadm/app/util/chroot_unix.go

@@ -0,0 +1,40 @@
+// +build !windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// Chroot chroot()s to the new path.
+// NB: All file paths after this call are effectively relative to
+// `rootfs`
+func Chroot(rootfs string) error {
+	if err := syscall.Chroot(rootfs); err != nil {
+		return fmt.Errorf("unable to chroot to %s: %v", rootfs, err)
+	}
+	root := filepath.FromSlash("/")
+	if err := os.Chdir(root); err != nil {
+		return fmt.Errorf("unable to chdir to %s: %v", root, err)
+	}
+	return nil
+}

cmd/kubeadm/app/util/chroot_windows.go

@@ -0,0 +1,30 @@
+// +build windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+)
+
+// Chroot chroot()s to the new path.
+// NB: All file paths after this call are effectively relative to
+// `rootfs`
+func Chroot(rootfs string) error {
+	return fmt.Errorf("chroot is not implemented on Windows")
+}