e2e node: make possible to add additional kernel arguments

Add an option to configure additional kernel arguments during
the setup of e2e node environment.

The example:
cos-stable1:
image_family: cos-89-lts # docker v19.03.6, deprecated after 2021-06-24
project: cos-cloud
metadata: "user-data<test/e2e_node/jenkins/cos-init-live-restore.yaml,gci-update-strategy=update_disabled"
kernel_arguments:
- "numa=fake=2"
machine: n1-standard-4

Signed-off-by: Artyom Lukianov <alukiano@redhat.com>

test/e2e_node/remote/ssh.go

@@ -110,8 +110,10 @@ func runSSHCommand(cmd string, args ...string) (string, error) {
 	if *sshOptions != "" {
 		args = append(strings.Split(*sshOptions, " "), args...)
 	}
+	klog.Infof("Running the command %s, with args: %v", cmd, args)
 	output, err := exec.Command(cmd, args...).CombinedOutput()
 	if err != nil {
+		klog.Errorf("failed to run SSH command: out: %s, err: %v", output, err)
 		return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err)
 	}
 	return string(output), nil

test/e2e_node/runner/remote/run_remote.go

@@ -36,6 +36,7 @@ import (
 	"sync"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e_node/remote"
 	"k8s.io/kubernetes/test/e2e_node/system"
 
@@ -154,12 +155,13 @@ type GCEImage struct {
 	Image      string `json:"image,omitempty"`
 	ImageRegex string `json:"image_regex,omitempty"`
 	// ImageFamily is the image family to use. The latest image from the image family will be used, e.g cos-81-lts.
-	ImageFamily string    `json:"image_family,omitempty"`
+	ImageFamily     string    `json:"image_family,omitempty"`
-	ImageDesc   string    `json:"image_description,omitempty"`
+	ImageDesc       string    `json:"image_description,omitempty"`
-	Project     string    `json:"project"`
+	KernelArguments []string  `json:"kernel_arguments,omitempty"`
-	Metadata    string    `json:"metadata"`
+	Project         string    `json:"project"`
-	Machine     string    `json:"machine,omitempty"`
+	Metadata        string    `json:"metadata"`
-	Resources   Resources `json:"resources,omitempty"`
+	Machine         string    `json:"machine,omitempty"`
+	Resources       Resources `json:"resources,omitempty"`
 	// This test is for benchmark (no limit verification, more result log, node name has format 'machine-image-uuid') if 'Tests' is non-empty.
 	Tests []string `json:"tests,omitempty"`
 }
@@ -173,12 +175,13 @@ type internalGCEImage struct {
 	image string
 	// imageDesc is the description of the image. If empty, the value in the
 	// 'image' will be used.
-	imageDesc string
+	imageDesc       string
-	project   string
+	kernelArguments []string
-	resources Resources
+	project         string
-	metadata  *compute.Metadata
+	resources       Resources
-	machine   string
+	metadata        *compute.Metadata
-	tests     []string
+	machine         string
+	tests           []string
 }
 
 func main() {
@@ -252,13 +255,14 @@ func main() {
 				metadata += "," + *instanceMetadata
 			}
 			gceImage := internalGCEImage{
-				image:     image,
+				image:           image,
-				imageDesc: imageConfig.ImageDesc,
+				imageDesc:       imageConfig.ImageDesc,
-				project:   imageConfig.Project,
+				project:         imageConfig.Project,
-				metadata:  getImageMetadata(metadata),
+				metadata:        getImageMetadata(metadata),
-				machine:   imageConfig.Machine,
+				kernelArguments: imageConfig.KernelArguments,
-				tests:     imageConfig.Tests,
+				machine:         imageConfig.Machine,
-				resources: imageConfig.Resources,
+				tests:           imageConfig.Tests,
+				resources:       imageConfig.Resources,
 			}
 			if gceImage.imageDesc == "" {
 				gceImage.imageDesc = gceImage.image
@@ -632,6 +636,7 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 		insertionOperationName = op.Name
 	}
 	instanceRunning := false
+	var instance *compute.Instance
 	for i := 0; i < 30 && !instanceRunning; i++ {
 		if i > 0 {
 			time.Sleep(time.Second * 20)
@@ -653,7 +658,6 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 			return name, fmt.Errorf("could not create instance %s: %+v", name, errs)
 		}
 
-		var instance *compute.Instance
 		instance, err = computeService.Instances.Get(*project, *zone, name).Do()
 		if err != nil {
 			continue
@@ -702,9 +706,84 @@ func createInstance(imageConfig *internalGCEImage) (string, error) {
 			cloudInitFinished = true
 		}
 	}
+
+	// apply additional kernel arguments to the instance
+	if len(imageConfig.kernelArguments) > 0 {
+		klog.Info("Update kernel arguments")
+		if err := updateKernelArguments(instance, imageConfig.image, imageConfig.kernelArguments); err != nil {
+			return name, err
+		}
+	}
+
 	return name, err
 }
 
+func updateKernelArguments(instance *compute.Instance, image string, kernelArgs []string) error {
+	kernelArgsString := strings.Join(kernelArgs, " ")
+
+	var cmd []string
+	if strings.Contains(image, "cos") {
+		cmd = []string{
+			"dir=$(mktemp -d)",
+			"mount /dev/sda12 ${dir}",
+			fmt.Sprintf("sed -i -e \"s|cros_efi|cros_efi %s|g\" ${dir}/efi/boot/grub.cfg", kernelArgsString),
+			"umount ${dir}",
+			"rmdir ${dir}",
+		}
+	}
+
+	if strings.Contains(image, "ubuntu") {
+		cmd = []string{
+			fmt.Sprintf("echo \"GRUB_CMDLINE_LINUX_DEFAULT=%s ${GRUB_CMDLINE_LINUX_DEFAULT}\" > /etc/default/grub.d/99-additional-arguments.cfg", kernelArgsString),
+			"/usr/sbin/update-grub",
+		}
+	}
+
+	if len(cmd) == 0 {
+		klog.Warningf("The image %s does not support adding an additional kernel arguments", image)
+		return nil
+	}
+
+	out, err := remote.SSH(instance.Name, "sh", "-c", fmt.Sprintf("'%s'", strings.Join(cmd, "&&")))
+	if err != nil {
+		klog.Errorf("failed to run command %s: out: %s, err: %v", cmd, out, err)
+		return err
+	}
+
+	if err := rebootInstance(instance); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func rebootInstance(instance *compute.Instance) error {
+	// wait until the instance will not response to SSH
+	klog.Info("Reboot the node and wait for instance not to be available via SSH")
+	if waitErr := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
+		if _, err := remote.SSH(instance.Name, "reboot"); err != nil {
+			return true, nil
+		}
+
+		return false, nil
+	}); waitErr != nil {
+		return fmt.Errorf("the instance %s still response to SSH: %v", instance.Name, waitErr)
+	}
+
+	// wait until the instance will response again to SSH
+	klog.Info("Wait for instance to be available via SSH")
+	if waitErr := wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) {
+		if _, err := remote.SSH(instance.Name, "sh", "-c", "date"); err != nil {
+			return false, nil
+		}
+		return true, nil
+	}); waitErr != nil {
+		return fmt.Errorf("the instance %s does not response to SSH: %v", instance.Name, waitErr)
+	}
+
+	return nil
+}
+
 func isCloudInitUsed(metadata *compute.Metadata) bool {
 	if metadata == nil {
 		return false