Merge pull request #101903 from cynepco3hahue/e2e_remote_kernel_args

e2e node: make possible to add additional kernel arguments

test/e2e_node/remote/ssh.go

@@ -110,8 +110,10 @@ func runSSHCommand(cmd string, args ...string) (string, error) {
 	if *sshOptions != "" {
 		args = append(strings.Split(*sshOptions, " "), args...)
 	}
+	klog.Infof("Running the command %s, with args: %v", cmd, args)
 	output, err := exec.Command(cmd, args...).CombinedOutput()
 	if err != nil {
+		klog.Errorf("failed to run SSH command: out: %s, err: %v", output, err)
 		return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err)
 	}
 	return string(output), nil

test/e2e_node/runner/remote/run_remote.go

@@ -36,6 +36,7 @@ import (
 	"sync"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e_node/remote"
 	"k8s.io/kubernetes/test/e2e_node/system"
 
@@ -156,6 +157,7 @@ type GCEImage struct {
 	// ImageFamily is the image family to use. The latest image from the image family will be used, e.g cos-81-lts.
 	ImageFamily     string    `json:"image_family,omitempty"`
 	ImageDesc       string    `json:"image_description,omitempty"`
+	KernelArguments []string  `json:"kernel_arguments,omitempty"`
 	Project         string    `json:"project"`
 	Metadata        string    `json:"metadata"`
 	Machine         string    `json:"machine,omitempty"`
@@ -174,6 +176,7 @@ type internalGCEImage struct {
 	// imageDesc is the description of the image. If empty, the value in the
 	// 'image' will be used.
 	imageDesc       string
+	kernelArguments []string
 	project         string
 	resources       Resources
 	metadata        *compute.Metadata
@@ -256,6 +259,7 @@ func main() {
 				imageDesc:       imageConfig.ImageDesc,
 				project:         imageConfig.Project,
 				metadata:        getImageMetadata(metadata),
+				kernelArguments: imageConfig.KernelArguments,
 				machine:         imageConfig.Machine,
 				tests:           imageConfig.Tests,
 				resources:       imageConfig.Resources,
@@ -632,6 +636,7 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 		insertionOperationName = op.Name
 	}
 	instanceRunning := false
+	var instance *compute.Instance
 	for i := 0; i < 30 && !instanceRunning; i++ {
 		if i > 0 {
 			time.Sleep(time.Second * 20)
@@ -653,7 +658,6 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 			return name, fmt.Errorf("could not create instance %s: %+v", name, errs)
 		}
 
-		var instance *compute.Instance
 		instance, err = computeService.Instances.Get(*project, *zone, name).Do()
 		if err != nil {
 			continue
@@ -702,7 +706,82 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 			cloudInitFinished = true
 		}
 	}
+
+	// apply additional kernel arguments to the instance
+	if len(imageConfig.kernelArguments) > 0 {
+		klog.Info("Update kernel arguments")
+		if err := updateKernelArguments(instance, imageConfig.image, imageConfig.kernelArguments); err != nil {
 			return name, err
+		}
+	}
+
+	return name, err
+}
+
+func updateKernelArguments(instance *compute.Instance, image string, kernelArgs []string) error {
+	kernelArgsString := strings.Join(kernelArgs, " ")
+
+	var cmd []string
+	if strings.Contains(image, "cos") {
+		cmd = []string{
+			"dir=$(mktemp -d)",
+			"mount /dev/sda12 ${dir}",
+			fmt.Sprintf("sed -i -e \"s|cros_efi|cros_efi %s|g\" ${dir}/efi/boot/grub.cfg", kernelArgsString),
+			"umount ${dir}",
+			"rmdir ${dir}",
+		}
+	}
+
+	if strings.Contains(image, "ubuntu") {
+		cmd = []string{
+			fmt.Sprintf("echo \"GRUB_CMDLINE_LINUX_DEFAULT=%s ${GRUB_CMDLINE_LINUX_DEFAULT}\" > /etc/default/grub.d/99-additional-arguments.cfg", kernelArgsString),
+			"/usr/sbin/update-grub",
+		}
+	}
+
+	if len(cmd) == 0 {
+		klog.Warningf("The image %s does not support adding an additional kernel arguments", image)
+		return nil
+	}
+
+	out, err := remote.SSH(instance.Name, "sh", "-c", fmt.Sprintf("'%s'", strings.Join(cmd, "&&")))
+	if err != nil {
+		klog.Errorf("failed to run command %s: out: %s, err: %v", cmd, out, err)
+		return err
+	}
+
+	if err := rebootInstance(instance); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func rebootInstance(instance *compute.Instance) error {
+	// wait until the instance will not response to SSH
+	klog.Info("Reboot the node and wait for instance not to be available via SSH")
+	if waitErr := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
+		if _, err := remote.SSH(instance.Name, "reboot"); err != nil {
+			return true, nil
+		}
+
+		return false, nil
+	}); waitErr != nil {
+		return fmt.Errorf("the instance %s still response to SSH: %v", instance.Name, waitErr)
+	}
+
+	// wait until the instance will response again to SSH
+	klog.Info("Wait for instance to be available via SSH")
+	if waitErr := wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) {
+		if _, err := remote.SSH(instance.Name, "sh", "-c", "date"); err != nil {
+			return false, nil
+		}
+		return true, nil
+	}); waitErr != nil {
+		return fmt.Errorf("the instance %s does not response to SSH: %v", instance.Name, waitErr)
+	}
+
+	return nil
 }
 
 func isCloudInitUsed(metadata *compute.Metadata) bool {