provider-kairos/tests/e2e/tests_suite_test.go

package mos

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/kairos-io/kairos-sdk/utils"
	"github.com/luthermonson/go-proxmox"
	process "github.com/mudler/go-processmanager"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
)

func TestSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "kairos E2E proxmox Test Suite")
}

func genToken() (string, error) {
	return utils.SH("/usr/bin/edgevpn -b -g")
}

var ControlVM *SSHConn

func waitTask(t *proxmox.Task, err error) {
	Expect(err).ToNot(HaveOccurred())
	Expect(t).ToNot(BeNil())
	err = t.Wait(1*time.Second, 10*time.Second)
	Expect(err).ToNot(HaveOccurred())
}

func deleteVMs(node *proxmox.Node) {
	fmt.Println("delete all")
	vms, err := node.VirtualMachines()
	if err != nil {
		Fail(err.Error())
	}

	// Delete all VMs running
	for _, vm := range vms {
		v, _ := node.VirtualMachine(int(vm.VMID))
		fmt.Println(v)
		d, _ := json.Marshal(v)
		fmt.Println(string(d))
		fmt.Printf("Deleting %s\n", vm.Name)

		t, err := vm.Stop()
		waitTask(t, err)

		t, err = vm.Delete()
		waitTask(t, err)
	}
}

var _ = AfterSuite(func() {
	node, _, _ := getNode()
	deleteVMs(node)
})

// accessors
func proxmoxISO() string {
	return os.Getenv("PROXMOX_ISO")
}

func isoName() string {
	return filepath.Base(proxmoxISO())
}

func locateISOFmt(s string) string {
	return fmt.Sprintf("local:iso/%s,media=cdrom", s)
}

var _ = BeforeSuite(func() {

	// We need to setup the router VM where we will connect to
	// First upload the ISOs needed (OS and cidata)
	node, client, err := getNode()
	if err != nil {
		Fail(err.Error())
	}
	storage, err := node.Storage(os.Getenv("PROXMOX_STORAGE"))
	if err != nil {
		Fail(err.Error())
	}

	temp, err := os.MkdirTemp("", "datasource")
	if err != nil {
		Fail(err.Error())
	}
	defer os.RemoveAll(temp)

	token, err := genToken()
	if err != nil {
		Fail(err.Error() + token)
	}
	fmt.Println("start vpn")

	fmt.Println("Upload cloud init")
	err = uploadCloudInitISO(
		"control.iso",
		[]byte(fmt.Sprintf(`#cloud-config

stages:
  boot.after:
  - commands:
    - EDGEVPNTOKEN=%s edgevpn service-add "ssh" "127.0.0.1:22"
users:
- name: "kairos"
  passwd: "kairos"

install:
  auto: true
  device: "auto"
  reboot: true
k3s:
  enable: true`, token)), storage,
	)
	if err != nil {
		Fail(err.Error())
	}

	iso, err := storage.ISO(isoName())
	if err != nil || iso == nil {
		fmt.Println("Upload target iso")
		tup, err := storage.Upload("iso", proxmoxISO())
		if err != nil {
			Fail(err.Error())
		}
		if err := tup.WaitFor(300); err != nil {
			Fail(err.Error())
		}
	}

	deleteVMs(node)

	// Create control VM and wait for it to be available
	cluster, err := client.Cluster()
	Expect(err).ToNot(HaveOccurred())

	nextid, err := cluster.NextID()
	Expect(err).ToNot(HaveOccurred())

	fmt.Println("Next ID", nextid)

	// See: https://pve.proxmox.com/pve-docs/api-viewer/index.html#/nodes/{node}/qemu
	t, err := node.NewVirtualMachine(nextid,
		proxmox.VirtualMachineOption{Name: "serial0", Value: "socket"},
		proxmox.VirtualMachineOption{Name: "memory", Value: "2048"},
		proxmox.VirtualMachineOption{Name: "cores", Value: "1"},
		proxmox.VirtualMachineOption{Name: "boot", Value: "order=scsi0;ide0;net0"},
		proxmox.VirtualMachineOption{Name: "net0", Value: "virtio,bridge=vmbr0,firewall=1"},
		proxmox.VirtualMachineOption{Name: "scsi0", Value: "local-lvm:60,size=60G"},
		proxmox.VirtualMachineOption{Name: "ide0", Value: locateISOFmt(isoName())},
		proxmox.VirtualMachineOption{Name: "ide1", Value: locateISOFmt("control.iso")},
	)
	Expect(err).ToNot(HaveOccurred())
	Expect(t).ToNot(BeNil())
	err = t.Wait(1*time.Second, 10*time.Second)
	Expect(err).ToNot(HaveOccurred())

	v, _ := node.VirtualMachine(nextid)
	v.Start()

	edgevpn := process.New(
		process.WithEnvironment(fmt.Sprintf("EDGEVPNTOKEN=%s", token)),
		process.WithName("/usr/bin/edgevpn"), process.WithArgs("service-connect", "--log-level", "debug", "ssh", "0.0.0.0:9090"), process.WithStateDir(temp))
	err = edgevpn.Run()

	// fmt.Println("start vpn", err)

	//defer edgevpn.Stop()
	ControlVM = NewSSH("kairos", "kairos", "127.0.0.1:9090")

	Eventually(func() string {
		out, err := ControlVM.Command("kairos-agent state get boot")
		if err != nil {
			fmt.Println(err)
		}
		return out
	}, time.Duration(time.Duration(920)*time.Second), time.Duration(30*time.Second)).Should(Equal("active_boot"))

	out, err := ControlVM.Command(`ssh-keygen -t rsa -q -f "$HOME/.ssh/id_rsa" -N ""`)
	Expect(err).ToNot(HaveOccurred(), out)
	fmt.Println(out)
})