provider-kairos/tests/e2e/ha_test.go

package mos_test

import (
	"fmt"
	"strings"
	"time"

	"github.com/luthermonson/go-proxmox"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/types"
)

func genConfig(kubeVIP string, rsa string, token string) string {
	return fmt.Sprintf(`#cloud-config
install:
  auto: true
  device: "auto"
  reboot: true
hostname: "test-{{ trunc 4 .MachineID }}"
users:
- name: "kairos"
  passwd: "kairos"
  ssh_authorized_keys:
  - "%s"
## Sets the Elastic IP for KubeVIP
kubevip:
 eip: "%s"
 manifest_url: ""
 enable: true

p2p:
 disable_dht: true # Enabled by default
 vpn:
  create: false # defaults to true
  use: false # defaults to true
 auto-ha:
  # Enables HA
  enable: true
  # number of HA/master node (beside the one used for init)
  master_nodes: 2
 network_token: "%s"`, rsa, kubeVIP, token)
}

var _ = Describe("kairos decentralized k8s test", Label("proxmox-ha-test"), func() {
	BeforeEach(func() {
		//	EventuallyConnects()
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			//		gatherLogs()
		}
	})

	Context("HA", func() {
		It("setups automatically an EmbeddedDB cluster", func() {

			// Get Possible IP to use for KubeVIP
			dev, err := ControlVM.Command(`ls -l /sys/class/net/ | grep -v virtual | cut -d " " -f9`)
			Expect(err).ToNot(HaveOccurred())
			fmt.Println("Found ControlVM dev at", dev)

			ip, err := ControlVM.Command(fmt.Sprintf(`ip a s %s | awk -F'[/ ]+' '/inet[^6]/{print $3}'`, strings.TrimSpace(dev)))
			Expect(err).ToNot(HaveOccurred(), ip)
			fmt.Println("Found ControlVM IP at", ip)

			pubkey, err := ControlVM.Command(`cat .ssh/id_rsa.pub`)
			Expect(err).ToNot(HaveOccurred())
			pubkey = strings.TrimSpace(pubkey)

			fmt.Println("Found ControlVM pubkey ", pubkey)

			freeIP, err := ControlVM.Command(fmt.Sprintf(`my_net=%s
			for i in $(seq 1 254);
			do
			 ip="$my_net.$i"
			 ping -c2 $ip | grep -q "Unreachable"; [[ "$?" == "0" ]] && echo $ip && break
			done`, strings.Join(strings.Split(ip, ".")[0:3], ".")))
			Expect(err).ToNot(HaveOccurred(), freeIP)

			freeIP = strings.TrimSpace(freeIP)
			fmt.Println("Found Free IP at", freeIP)

			r := RandStringRunes(5)
			isoTestName := fmt.Sprintf("%s.iso", r)
			node, client, err := getNode()
			Expect(err).ToNot(HaveOccurred())

			storage, err := node.Storage("local")
			Expect(err).ToNot(HaveOccurred())

			networkToken, err := genToken()
			Expect(err).ToNot(HaveOccurred())

			err = uploadCloudInitISO(
				isoTestName,
				[]byte(genConfig(freeIP, pubkey, networkToken)), storage,
			)
			Expect(err).ToNot(HaveOccurred())
			// ip a s eth0 | awk -F'[/ ]+' '/inet[^6]/{print $3}'
			cluster, err := client.Cluster()
			Expect(err).ToNot(HaveOccurred())

			for i := 0; i < 4; i++ {
				nextid, err := cluster.NextID()
				Expect(err).ToNot(HaveOccurred())

				fmt.Println("Next ID", nextid)

				// See: https://pve.proxmox.com/pve-docs/api-viewer/index.html#/nodes/{node}/qemu
				t, err := node.NewVirtualMachine(nextid,
					proxmox.VirtualMachineOption{Name: "serial0", Value: "socket"},
					proxmox.VirtualMachineOption{Name: "memory", Value: "2048"},
					proxmox.VirtualMachineOption{Name: "cores", Value: "1"},
					proxmox.VirtualMachineOption{Name: "boot", Value: "order=scsi0;ide0;net0"},
					proxmox.VirtualMachineOption{Name: "net0", Value: "virtio,bridge=vmbr0,firewall=1"},
					proxmox.VirtualMachineOption{Name: "scsi0", Value: "local-lvm:60,size=60G"},
					proxmox.VirtualMachineOption{Name: "ide0", Value: locateISOFmt(isoName())},
					proxmox.VirtualMachineOption{Name: "ide1", Value: locateISOFmt(isoTestName)},
				)
				Expect(err).ToNot(HaveOccurred())
				Expect(t).ToNot(BeNil())

				err = t.Wait(1*time.Second, 10*time.Second)
				Expect(err).ToNot(HaveOccurred())

				v, _ := node.VirtualMachine(nextid)
				v.Start()
			}

			By("Waiting for HA control-plane to be available", func() {
				Eventually(func() string {
					out, err := ControlVM.Command(fmt.Sprintf("ping %s -c 3", freeIP))
					if err != nil {
						fmt.Println(err)
					}
					return out
				}, time.Duration(time.Duration(650)*time.Second), time.Duration(30*time.Second)).Should(ContainSubstring("3 received"))
			})

			Eventually(func() string {
				out, err := ControlVM.Command(fmt.Sprintf("ssh -oStrictHostKeyChecking=no kairos@%s kairos role list", freeIP))
				if err != nil {
					fmt.Println(err, out)
				}
				return out
			}, time.Duration(time.Duration(650)*time.Second), time.Duration(30*time.Second)).Should(And(
				ContainSubstring("worker"),
				ContainSubstring("master/ha"),
				ContainSubstring("master/clusterinit"),
				HaveMinMaxRole("master/clusterinit", 1, 1),
				HaveMinMaxRole("master/ha", 2, 2),
			))
		})
	})
})

func HaveMinMaxRole(name string, min, max int) types.GomegaMatcher {
	return WithTransform(
		func(actual interface{}) (int, error) {
			switch s := actual.(type) {
			case string:
				return strings.Count(s, name), nil
			default:
				return 0, fmt.Errorf("HaveRoles expects a string, but got %T", actual)
			}
		}, SatisfyAll(
			BeNumerically(">=", min),
			BeNumerically("<=", max)))
}