From 6fdb20480fb904403024563dc2e44a605f1ccc5e Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Wed, 19 Oct 2016 13:50:36 -0700 Subject: [PATCH 1/6] Bundle GCI mounter w/ node tests and plumb --mounter-path through test args to the Kubelet for node tests --- test/e2e/framework/test_context.go | 3 ++ test/e2e_node/build/build.go | 6 +-- test/e2e_node/remote/remote.go | 66 +++++++++++++++++++++++++++++- test/e2e_node/services/services.go | 2 + 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 11495028796..4f4689ced4f 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -114,6 +114,8 @@ type NodeTestContextType struct { PrepullImages bool // RuntimeIntegrationType indicates how runtime is integrated with Kubelet. This is mainly used for CRI validation test. RuntimeIntegrationType string + // MounterPath is the path to the program to run to perform a mount + MounterPath string } type CloudConfig struct { @@ -209,6 +211,7 @@ func RegisterNodeFlags() { flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.") flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.") + flag.StringVar(&TestContext.MounterPath, "mounter-path", "", "Path of mounter binary. Leave empty to use the default mount.") } // overwriteFlagsWithViperConfig finds and writes values to flags using viper as input. diff --git a/test/e2e_node/build/build.go b/test/e2e_node/build/build.go index b54d8b64119..d80f6a132fb 100644 --- a/test/e2e_node/build/build.go +++ b/test/e2e_node/build/build.go @@ -38,7 +38,7 @@ var buildTargets = []string{ func BuildGo() error { glog.Infof("Building k8s binaries...") - k8sRoot, err := getK8sRootDir() + k8sRoot, err := GetK8sRootDir() if err != nil { return fmt.Errorf("failed to locate kubernetes root directory %v.", err) } @@ -87,7 +87,7 @@ func getK8sBin(bin string) (string, error) { } // TODO: Dedup / merge this with comparable utilities in e2e/util.go -func getK8sRootDir() (string, error) { +func GetK8sRootDir() (string, error) { // Get the directory of the current executable _, testExec, _, _ := runtime.Caller(0) path := filepath.Dir(testExec) @@ -102,7 +102,7 @@ func getK8sRootDir() (string, error) { } func GetK8sBuildOutputDir() (string, error) { - k8sRoot, err := getK8sRootDir() + k8sRoot, err := GetK8sRootDir() if err != nil { return "", err } diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index d056752ad32..c10e9b07ce0 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -113,8 +113,34 @@ func CreateTestArchive() (string, error) { } } + // Include the GCI mounter in the deployed tarball + k8sDir, err := build.GetK8sRootDir() + if err != nil { + return "", fmt.Errorf("Could not find K8s root dir! Err: %v", err) + } + localSource := "cluster/gce/gci/mounter/mounter" + source := filepath.Join(k8sDir, localSource) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err := os.Stat(source); err != nil { + return "", fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + bindir := "cluster/gce/gci/mounter" + bin := "mounter" + destdir := filepath.Join(tardir, bindir) + dest := filepath.Join(destdir, bin) + out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out) + } + out, err = exec.Command("cp", source, dest).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out) + } + // Build the tar - out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() + out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() if err != nil { return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) } @@ -213,6 +239,44 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Exit failure with the error return "", false, err } + + // If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args. + // We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run. + + // Determine if the GCI mounter script exists locally. + k8sDir, err := build.GetK8sRootDir() + if err != nil { + return "", false, fmt.Errorf("Could not find K8s root dir! Err: %v", err) + } + localSource := "cluster/gce/gci/mounter/mounter" + source := filepath.Join(k8sDir, localSource) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err = os.Stat(source); err != nil { + return "", false, fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + // Determine if tests will run on a GCI node. + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", "'cat /etc/os-release'") + if err != nil { + glog.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) + return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) + } + if strings.Contains(output, "ID=gci") { + glog.Infof("GCI node and GCI mounter both detected, modifying --mounter-path accordingly") + + // Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error + // here will tell us to update the remote test runner. + mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter") + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath)) + if err != nil { + glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output) + return "", false, err + } + // Insert args at beginning of testArgs, so any values from command line take precedence + testArgs = fmt.Sprintf("--mounter-path=%s ", mounterPath) + testArgs + } + // Run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 7fa3aa85e6e..7838f3e0b64 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -211,7 +211,9 @@ func (e *E2EServices) startKubelet() (*server, error) { "--eviction-pressure-transition-period", "30s", "--feature-gates", framework.TestContext.FeatureGates, "--v", LOG_VERBOSITY_LEVEL, "--logtostderr", + "--mounter-path", framework.TestContext.MounterPath, ) + if framework.TestContext.RuntimeIntegrationType != "" { cmdArgs = append(cmdArgs, "--experimental-runtime-integration-type", framework.TestContext.RuntimeIntegrationType) // Whether to use experimental cri integration. From f819cada9c658de3b981be27868de6fc70f032ad Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Thu, 20 Oct 2016 14:32:23 -0700 Subject: [PATCH 2/6] Add a bare-bones level of indirection for GCI mounter This is the bare bones scaffolding from @vishh's PR #34787 --- build/lib/release.sh | 1 + cluster/gce/gci/configure-helper.sh | 1 + cluster/gce/gci/configure.sh | 3 ++- cluster/gce/gci/master.yaml | 1 + cluster/gce/gci/mounter/mounter | 20 ++++++++++++++++++++ cluster/gce/gci/node.yaml | 1 + 6 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 cluster/gce/gci/mounter/mounter diff --git a/build/lib/release.sh b/build/lib/release.sh index fa8b34c3d75..318aba2ccf4 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -320,6 +320,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/" cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh" + cp "${KUBE_ROOT}/cluster/gce/gci/mounter" "${dst_dir}/gci-mounter" cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}" local objects diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b4d3e0ce84d..776d92a211a 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -475,6 +475,7 @@ function start-kubelet { flags+=" --config=/etc/kubernetes/manifests" flags+=" --kubelet-cgroups=/kubelet" flags+=" --system-cgroups=/system" + flags+=" --mounter-path=${KUBE_HOME}/bin/mounter" if [[ -n "${KUBELET_PORT:-}" ]]; then flags+=" --port=${KUBELET_PORT}" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index fcd9e3e9743..8c59ca2ac67 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -99,7 +99,7 @@ function split-commas { } # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, -# and places them into suitable directories. Files are placed in /home/kubernetes. +# and places them into suitable directories. Files are placed in /home/kubernetes. function install-kube-binary-config { cd "${KUBE_HOME}" local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") ) @@ -171,6 +171,7 @@ function install-kube-binary-config { xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" fi cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" + cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter" cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh" chmod -R 755 "${kube_bin}" diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml index 845973df84b..78b39164ae4 100644 --- a/cluster/gce/gci/master.yaml +++ b/cluster/gce/gci/master.yaml @@ -34,6 +34,7 @@ write_files: Type=oneshot RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] diff --git a/cluster/gce/gci/mounter/mounter b/cluster/gce/gci/mounter/mounter new file mode 100644 index 00000000000..78ad6c0d424 --- /dev/null +++ b/cluster/gce/gci/mounter/mounter @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -o pipefail + +sudo /bin/mount "$@" \ No newline at end of file diff --git a/cluster/gce/gci/node.yaml b/cluster/gce/gci/node.yaml index de59fcac27b..d0cc12e752d 100644 --- a/cluster/gce/gci/node.yaml +++ b/cluster/gce/gci/node.yaml @@ -34,6 +34,7 @@ write_files: Type=oneshot RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] From dba917c5b782787247b895ed3e4cb00f9b88532f Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Tue, 18 Oct 2016 12:35:58 -0700 Subject: [PATCH 3/6] Include mount command in Kubelet mounter output --- pkg/util/mount/mount_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 265915dd63a..0ea855c5397 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -101,15 +101,15 @@ func isBind(options []string) (bool, []string) { // doMount runs the mount command. func doMount(mountCmd string, source string, target string, fstype string, options []string) error { - glog.V(5).Infof("Mounting %s %s %s %v", source, target, fstype, options) + glog.V(5).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd) mountArgs := makeMountArgs(source, target, fstype, options) command := exec.Command(mountCmd, mountArgs...) output, err := command.CombinedOutput() if err != nil { - glog.Errorf("Mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, source, target, fstype, options, string(output)) - return fmt.Errorf("mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n", - err, source, target, fstype, options, string(output)) + glog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, mountCmd, source, target, fstype, options, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n", + err, mountCmd, source, target, fstype, options, string(output)) } return err } From c339c975838a854d77e00c51a5e3a4dd97a834da Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Tue, 18 Oct 2016 15:05:34 -0700 Subject: [PATCH 4/6] Simple mount test --- test/e2e_node/simple_mount.go | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 test/e2e_node/simple_mount.go diff --git a/test/e2e_node/simple_mount.go b/test/e2e_node/simple_mount.go new file mode 100644 index 00000000000..5bcbe5e8ab6 --- /dev/null +++ b/test/e2e_node/simple_mount.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" +) + +var _ = framework.KubeDescribe("SimpleMount", func() { + f := framework.NewDefaultFramework("simple-mount-test") + + // This is a very simple test that exercises the Kubelet's mounter code path. + // If the mount fails, the pod will not be able to run, and CreateSync will timeout. + It("should be able to mount an emptydir on a container", func() { + pod := &api.Pod{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: api.ObjectMeta{ + Name: "simple-mount-pod", + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "simple-mount-container", + Image: framework.GetPauseImageNameForHostArch(), + VolumeMounts: []api.VolumeMount{ + { + Name: "simply-mounted-volume", + MountPath: "/opt/", + }, + }, + }, + }, + Volumes: []api.Volume{ + { + Name: "simply-mounted-volume", + VolumeSource: api.VolumeSource{ + EmptyDir: &api.EmptyDirVolumeSource{ + Medium: "Memory", + }, + }, + }, + }, + }, + } + podClient := f.PodClient() + pod = podClient.CreateSync(pod) + + }) +}) From d5b1b4087e846f11f03b8510c835c2b37efe9b56 Mon Sep 17 00:00:00 2001 From: Vish Kannan Date: Fri, 21 Oct 2016 16:22:32 -0700 Subject: [PATCH 5/6] Fix path to gci mounter in release.sh --- build/lib/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/lib/release.sh b/build/lib/release.sh index 318aba2ccf4..35e3a28d231 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -320,7 +320,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/" cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh" - cp "${KUBE_ROOT}/cluster/gce/gci/mounter" "${dst_dir}/gci-mounter" + cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter" cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}" local objects From a8db72c4a3896c1966574949065e23bd835a79af Mon Sep 17 00:00:00 2001 From: Vish Kannan Date: Fri, 21 Oct 2016 17:37:42 -0700 Subject: [PATCH 6/6] Do not enable the new mounter on GCI nodes --- cluster/gce/gci/configure-helper.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 776d92a211a..b4d3e0ce84d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -475,7 +475,6 @@ function start-kubelet { flags+=" --config=/etc/kubernetes/manifests" flags+=" --kubelet-cgroups=/kubelet" flags+=" --system-cgroups=/system" - flags+=" --mounter-path=${KUBE_HOME}/bin/mounter" if [[ -n "${KUBELET_PORT:-}" ]]; then flags+=" --port=${KUBELET_PORT}"