Merge pull request #60342 from joelsmith/nested

Automatic merge from submit-queue (batch tested with PRs 60342, 60505, 59218, 52900, 60486). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix nested volume mounts for read-only API data volumes

**What this PR does / why we need it**:

Since the runtime may try to create mount points within the sandbox, it will fail if the mount point is within a read-only API data volume, like a secret or configMap volume.

Create any needed mount points during volume setup.

**Which issue(s) this PR fixes**:
Fixes #60341

**Special notes for your reviewer**:


**Release note**:

No release note needed since we haven't released any versions with this bug present (yet)

```release-note
NONE

```

pkg/volume/configmap/configmap.go

@@ -194,6 +194,9 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 	if err := wrapped.SetUpAt(dir, fsGroup); err != nil {
 		return err
 	}
+	if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil {
+		return err
+	}
 
 	optional := b.source.Optional != nil && *b.source.Optional
 	configMap, err := b.getConfigMap(b.pod.Namespace, b.source.Name)

pkg/volume/downwardapi/downwardapi.go

@@ -183,6 +183,9 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 		glog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
 		return err
 	}
+	if err := volumeutil.MakeNestedMountpoints(b.volName, dir, *b.pod); err != nil {
+		return err
+	}
 
 	data, err := CollectData(b.source.Items, b.pod, b.plugin.host, b.source.DefaultMode)
 	if err != nil {

pkg/volume/projected/projected.go

@@ -191,6 +191,9 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 	if err := wrapped.SetUpAt(dir, fsGroup); err != nil {
 		return err
 	}
+	if err := volumeutil.MakeNestedMountpoints(s.volName, dir, *s.pod); err != nil {
+		return err
+	}
 
 	data, err := s.collectData()
 	if err != nil {

pkg/volume/secret/secret.go

@@ -193,6 +193,9 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 	if err := wrapped.SetUpAt(dir, fsGroup); err != nil {
 		return err
 	}
+	if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil {
+		return err
+	}
 
 	optional := b.source.Optional != nil && *b.source.Optional
 	secret, err := b.getSecret(b.pod.Namespace, b.source.SecretName)

pkg/volume/util/BUILD

@@ -10,6 +10,7 @@ go_library(
         "finalizer.go",
         "io_util.go",
         "metrics.go",
+        "nested_volumes.go",
         "resize_util.go",
         "util.go",
     ] + select({
@@ -77,6 +78,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "nested_volumes_test.go",
         "resize_util_test.go",
         "util_test.go",
     ] + select({
@@ -96,6 +98,7 @@ go_test(
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/client-go/util/testing:go_default_library",
     ],

pkg/volume/util/nested_volumes.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"k8s.io/api/core/v1"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// getNestedMountpoints returns a list of mountpoint directories that should be created
+// for the volume indicated by name.
+// note: the returned list is relative to baseDir
+func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) {
+	var retval []string
+	checkContainer := func(container *v1.Container) error {
+		var allMountPoints []string // all mount points in this container
+		var myMountPoints []string  // mount points that match name
+		for _, vol := range container.VolumeMounts {
+			cleaned := filepath.Clean(vol.MountPath)
+			allMountPoints = append(allMountPoints, cleaned)
+			if vol.Name == name {
+				myMountPoints = append(myMountPoints, cleaned)
+			}
+		}
+		sort.Strings(allMountPoints)
+		parentPrefix := ".." + string(os.PathSeparator)
+		// Examine each place where this volume is mounted
+		for _, myMountPoint := range myMountPoints {
+			if strings.HasPrefix(myMountPoint, parentPrefix) {
+				// Don't let a container trick us into creating directories outside of its rootfs
+				return fmt.Errorf("Invalid container mount point %v", myMountPoint)
+			}
+			myMPSlash := myMountPoint + string(os.PathSeparator)
+			// The previously found nested mountpoint (or "" if none found yet)
+			prevNestedMP := ""
+			// examine each mount point to see if it's nested beneath this volume
+			// (but skip any that are double-nested beneath this volume)
+			// For example, if this volume is mounted as /dir and other volumes are mounted
+			//              as /dir/nested and /dir/nested/other, only create /dir/nested.
+			for _, mp := range allMountPoints {
+				if !strings.HasPrefix(mp, myMPSlash) {
+					continue // skip -- not nested beneath myMountPoint
+				}
+				if prevNestedMP != "" && strings.HasPrefix(mp, prevNestedMP) {
+					continue // skip -- double nested beneath myMountPoint
+				}
+				// since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one
+				prevNestedMP = mp + string(os.PathSeparator)
+				retval = append(retval, mp[len(myMPSlash):])
+			}
+		}
+		return nil
+	}
+	for _, container := range pod.Spec.InitContainers {
+		if err := checkContainer(&container); err != nil {
+			return nil, err
+		}
+	}
+	for _, container := range pod.Spec.Containers {
+		if err := checkContainer(&container); err != nil {
+			return nil, err
+		}
+	}
+	return retval, nil
+}
+
+// MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name
+func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error {
+	dirs, err := getNestedMountpoints(name, baseDir, pod)
+	if err != nil {
+		return err
+	}
+	for _, dir := range dirs {
+		err := os.MkdirAll(path.Join(baseDir, dir), 0755)
+		if err != nil {
+			return fmt.Errorf("Unable to create nested volume mountpoints: %v", err)
+		}
+	}
+	return nil
+}

pkg/volume/util/nested_volumes_test.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+type testCases struct {
+	name     string
+	err      bool
+	expected sets.String
+	volname  string
+	pod      v1.Pod
+}
+
+func TestGetNestedMountpoints(t *testing.T) {
+	var (
+		testNamespace = "test_namespace"
+		testPodUID    = types.UID("test_pod_uid")
+	)
+
+	tc := []testCases{
+		{
+			name:     "Simple Pod",
+			err:      false,
+			expected: sets.NewString(),
+			volname:  "vol1",
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/dir", Name: "vol1"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Simple Nested Pod",
+			err:      false,
+			expected: sets.NewString("nested"),
+			volname:  "vol1",
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/dir", Name: "vol1"},
+								{MountPath: "/dir/nested", Name: "vol2"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Unsorted Nested Pod",
+			err:      false,
+			expected: sets.NewString("nested", "nested2"),
+			volname:  "vol1",
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/dir/nested/double", Name: "vol3"},
+								{MountPath: "/ignore", Name: "vol4"},
+								{MountPath: "/dir/nested", Name: "vol2"},
+								{MountPath: "/ignore2", Name: "vol5"},
+								{MountPath: "/dir", Name: "vol1"},
+								{MountPath: "/dir/nested2", Name: "vol3"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Multiple vol1 mounts Pod",
+			err:      false,
+			expected: sets.NewString("nested", "nested2"),
+			volname:  "vol1",
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/dir", Name: "vol1"},
+								{MountPath: "/dir/nested", Name: "vol2"},
+								{MountPath: "/ignore", Name: "vol4"},
+								{MountPath: "/other", Name: "vol1"},
+								{MountPath: "/other/nested2", Name: "vol3"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Big Pod",
+			err:      false,
+			volname:  "vol1",
+			expected: sets.NewString("sub1/sub2/sub3", "sub1/sub2/sub4", "sub1/sub2/sub6", "sub"),
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/mnt", Name: "vol1"},
+								{MountPath: "/ignore", Name: "vol2"},
+								{MountPath: "/mnt/sub1/sub2/sub3", Name: "vol3"},
+								{MountPath: "/mnt/sub1/sub2/sub4", Name: "vol4"},
+								{MountPath: "/mnt/sub1/sub2/sub4/skip", Name: "vol5"},
+								{MountPath: "/mnt/sub1/sub2/sub4/skip2", Name: "vol5a"},
+								{MountPath: "/mnt/sub1/sub2/sub6", Name: "vol6"},
+								{MountPath: "/mnt7", Name: "vol7"},
+							},
+						},
+					},
+					InitContainers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "/mnt/dir", Name: "vol1"},
+								{MountPath: "/mnt/dir_ignore", Name: "vol8"},
+								{MountPath: "/ignore", Name: "vol9"},
+								{MountPath: "/mnt/dir/sub", Name: "vol11"},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:     "Naughty Pod",
+			err:      true,
+			expected: nil,
+			volname:  "vol1",
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: testNamespace,
+					UID:       testPodUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							VolumeMounts: []v1.VolumeMount{
+								{MountPath: "foo/../../dir", Name: "vol1"},
+								{MountPath: "foo/../../dir/skip", Name: "vol10"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tc {
+		dir, err := ioutil.TempDir("", "TestMakeNestedMountpoints.")
+		if err != nil {
+			t.Errorf("Unexpected error trying to create temp directory: %v", err)
+			return
+		}
+		defer os.RemoveAll(dir)
+
+		rootdir := path.Join(dir, "vol")
+		err = os.Mkdir(rootdir, 0755)
+		if err != nil {
+			t.Errorf("Unexpected error trying to create temp root directory: %v", err)
+			return
+		}
+
+		dirs, err := getNestedMountpoints(test.volname, rootdir, test.pod)
+		if test.err {
+			if err == nil {
+				t.Errorf("%v: expected error, got nil", test.name)
+			}
+			continue
+		} else {
+			if err != nil {
+				t.Errorf("%v: expected no error, got %v", test.name, err)
+				continue
+			}
+		}
+		actual := sets.NewString(dirs...)
+		if !test.expected.Equal(actual) {
+			t.Errorf("%v: unexpected nested directories created:\nexpected: %v\n     got: %v", test.name, test.expected, actual)
+		}
+	}
+}