diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 11495028796..4f4689ced4f 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -114,6 +114,8 @@ type NodeTestContextType struct { PrepullImages bool // RuntimeIntegrationType indicates how runtime is integrated with Kubelet. This is mainly used for CRI validation test. RuntimeIntegrationType string + // MounterPath is the path to the program to run to perform a mount + MounterPath string } type CloudConfig struct { @@ -209,6 +211,7 @@ func RegisterNodeFlags() { flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.") flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.") + flag.StringVar(&TestContext.MounterPath, "mounter-path", "", "Path of mounter binary. Leave empty to use the default mount.") } // overwriteFlagsWithViperConfig finds and writes values to flags using viper as input. diff --git a/test/e2e_node/build/build.go b/test/e2e_node/build/build.go index b54d8b64119..d80f6a132fb 100644 --- a/test/e2e_node/build/build.go +++ b/test/e2e_node/build/build.go @@ -38,7 +38,7 @@ var buildTargets = []string{ func BuildGo() error { glog.Infof("Building k8s binaries...") - k8sRoot, err := getK8sRootDir() + k8sRoot, err := GetK8sRootDir() if err != nil { return fmt.Errorf("failed to locate kubernetes root directory %v.", err) } @@ -87,7 +87,7 @@ func getK8sBin(bin string) (string, error) { } // TODO: Dedup / merge this with comparable utilities in e2e/util.go -func getK8sRootDir() (string, error) { +func GetK8sRootDir() (string, error) { // Get the directory of the current executable _, testExec, _, _ := runtime.Caller(0) path := filepath.Dir(testExec) @@ -102,7 +102,7 @@ func getK8sRootDir() (string, error) { } func GetK8sBuildOutputDir() (string, error) { - k8sRoot, err := getK8sRootDir() + k8sRoot, err := GetK8sRootDir() if err != nil { return "", err } diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index d056752ad32..c10e9b07ce0 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -113,8 +113,34 @@ func CreateTestArchive() (string, error) { } } + // Include the GCI mounter in the deployed tarball + k8sDir, err := build.GetK8sRootDir() + if err != nil { + return "", fmt.Errorf("Could not find K8s root dir! Err: %v", err) + } + localSource := "cluster/gce/gci/mounter/mounter" + source := filepath.Join(k8sDir, localSource) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err := os.Stat(source); err != nil { + return "", fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + bindir := "cluster/gce/gci/mounter" + bin := "mounter" + destdir := filepath.Join(tardir, bindir) + dest := filepath.Join(destdir, bin) + out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out) + } + out, err = exec.Command("cp", source, dest).CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out) + } + // Build the tar - out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() + out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() if err != nil { return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) } @@ -213,6 +239,44 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Exit failure with the error return "", false, err } + + // If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args. + // We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run. + + // Determine if the GCI mounter script exists locally. + k8sDir, err := build.GetK8sRootDir() + if err != nil { + return "", false, fmt.Errorf("Could not find K8s root dir! Err: %v", err) + } + localSource := "cluster/gce/gci/mounter/mounter" + source := filepath.Join(k8sDir, localSource) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err = os.Stat(source); err != nil { + return "", false, fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + // Determine if tests will run on a GCI node. + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", "'cat /etc/os-release'") + if err != nil { + glog.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) + return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) + } + if strings.Contains(output, "ID=gci") { + glog.Infof("GCI node and GCI mounter both detected, modifying --mounter-path accordingly") + + // Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error + // here will tell us to update the remote test runner. + mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter") + output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath)) + if err != nil { + glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output) + return "", false, err + } + // Insert args at beginning of testArgs, so any values from command line take precedence + testArgs = fmt.Sprintf("--mounter-path=%s ", mounterPath) + testArgs + } + // Run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 7fa3aa85e6e..7838f3e0b64 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -211,7 +211,9 @@ func (e *E2EServices) startKubelet() (*server, error) { "--eviction-pressure-transition-period", "30s", "--feature-gates", framework.TestContext.FeatureGates, "--v", LOG_VERBOSITY_LEVEL, "--logtostderr", + "--mounter-path", framework.TestContext.MounterPath, ) + if framework.TestContext.RuntimeIntegrationType != "" { cmdArgs = append(cmdArgs, "--experimental-runtime-integration-type", framework.TestContext.RuntimeIntegrationType) // Whether to use experimental cri integration.