Merge pull request #109 from kairos-io/hybrid

✨ Hybrid P2P w/kube-vip

Earthfile

@@ -124,7 +124,7 @@ docker:
         && INSTALL_K3S_SKIP_START="true" INSTALL_K3S_SKIP_ENABLE="true" INSTALL_K3S_SKIP_SELINUX_RPM="true" bash installer.sh \
         && INSTALL_K3S_SKIP_START="true" INSTALL_K3S_SKIP_ENABLE="true" INSTALL_K3S_SKIP_SELINUX_RPM="true" bash installer.sh agent \
         && rm -rf installer.sh
-    RUN luet install -y utils/edgevpn utils/k9s utils/nerdctl container/kubectl && luet cleanup
+    RUN luet install -y utils/edgevpn utils/k9s utils/nerdctl container/kubectl utils/kube-vip && luet cleanup
     # Drop env files from k3s as we will generate them
     IF [ -e "/etc/rancher/k3s/k3s.env" ]
         RUN rm -rf /etc/rancher/k3s/k3s.env /etc/rancher/k3s/k3s-agent.env && touch /etc/rancher/k3s/.keep

go.mod

@@ -1,6 +1,6 @@
 module github.com/kairos-io/provider-kairos
 
-go 1.19
+go 1.18
 
 replace github.com/elastic/gosigar => github.com/mudler/gosigar v0.14.3-0.20220502202347-34be910bdaaf
 
@@ -9,7 +9,7 @@ require (
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/google/go-containerregistry v0.11.0
 	github.com/ipfs/go-log v1.0.5
-	github.com/kairos-io/kairos v1.24.3-56.0.20221128123446-ab088839ec95
+	github.com/kairos-io/kairos v1.3.1
 	github.com/mudler/edgevpn v0.15.3
 	github.com/mudler/go-nodepair v0.0.0-20220507212557-7d47aa3cc1f1
 	github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3
@@ -88,8 +88,8 @@ require (
 	github.com/ipfs/go-ipns v0.1.2 // indirect
 	github.com/ipfs/go-log/v2 v2.5.1 // indirect
 	github.com/ipld/go-ipld-prime v0.16.0 // indirect
-	github.com/itchyny/gojq v0.12.9 // indirect
-	github.com/itchyny/timefmt-go v0.1.4 // indirect
+	github.com/itchyny/gojq v0.12.10 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect

go.sum

@@ -693,11 +693,11 @@ github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa9
 github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA=
 github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
 github.com/itchyny/gojq v0.12.8/go.mod h1:gE2kZ9fVRU0+JAksaTzjIlgnCa2akU+a1V0WXgJQN5c=
-github.com/itchyny/gojq v0.12.9 h1:biKpbKwMxVYhCU1d6mR7qMr3f0Hn9F5k5YykCVb3gmM=
-github.com/itchyny/gojq v0.12.9/go.mod h1:T4Ip7AETUXeGpD+436m+UEl3m3tokRgajd5pRfsR5oE=
+github.com/itchyny/gojq v0.12.10 h1:6TcS0VYWS6wgntpF/4tnrzwdCMjiTxRAxIqZWfDsDQU=
+github.com/itchyny/gojq v0.12.10/go.mod h1:o3FT8Gkbg/geT4pLI0tF3hvip5F3Y/uskjRz9OYa38g=
 github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
-github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM=
-github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
+github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
+github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -749,8 +749,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kairos-io/kairos v1.24.3-56.0.20221128123446-ab088839ec95 h1:dSTM/gePf/Czu7eOKwvv3J91AWMxH/Y4xOGVQNMu+OY=
-github.com/kairos-io/kairos v1.24.3-56.0.20221128123446-ab088839ec95/go.mod h1:Rgn/1YTvcTQIdtzvKT96FoMH8tdGiYI9tZwcu1CV1dI=
+github.com/kairos-io/kairos v1.3.1 h1:t9q4FnBFxAlOU8yFU/s/df+hhJyZQKAHb4944jeItHs=
+github.com/kairos-io/kairos v1.3.1/go.mod h1:HhMh4lGr5gCXRukRI1+y/P8ZwB1+VWYaUOMP1nRoa0o=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o=
 github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4=

internal/assets/assets.go

@@ -0,0 +1,18 @@
+package assets
+
+import (
+	"embed"
+	"io/fs"
+)
+
+//go:embed static
+var staticFiles embed.FS
+
+func GetStaticFS() fs.FS {
+	fsys, err := fs.Sub(staticFiles, "static")
+	if err != nil {
+		panic(err)
+	}
+
+	return fs.FS(fsys)
+}

internal/assets/static/kube_vip_rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services", "services/status", "nodes", "endpoints"]
+    verbs: ["list","get","watch", "update"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

internal/provider/bootstrap.go

@@ -19,6 +19,8 @@ import (
 	sdk "github.com/kairos-io/kairos/sdk/bus"
 	providerConfig "github.com/kairos-io/provider-kairos/internal/provider/config"
 	"github.com/kairos-io/provider-kairos/internal/role"
+	p2p "github.com/kairos-io/provider-kairos/internal/role/p2p"
+
 	"github.com/kairos-io/provider-kairos/internal/services"
 
 	"github.com/kairos-io/kairos/pkg/config"
@@ -95,11 +97,20 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
 		return ErrorEvent("No network token provided, exiting")
 	}
 
+	if !providerConfig.Kairos.Hybrid {
 		logger.Info("Configuring VPN")
 		if err := SetupVPN(services.EdgeVPNDefaultInstance, cfg.APIAddress, "/", true, providerConfig); err != nil {
 			return ErrorEvent("Failed setup VPN: %s", err.Error())
 		}
 
+	} else {
+		logger.Info("Configuring API")
+		if err := SetupAPI(cfg.APIAddress, "/", true, providerConfig); err != nil {
+			return ErrorEvent("Failed setup VPN: %s", err.Error())
+		}
+
+	}
+
 	networkID := "kairos"
 
 	if providerConfig.Kairos != nil && providerConfig.Kairos.NetworkID != "" {
@@ -120,11 +131,11 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
 		service.WithRoles(
 			service.RoleKey{
 				Role:        "master",
-				RoleHandler: role.Master(c, providerConfig),
+				RoleHandler: p2p.Master(c, providerConfig),
 			},
 			service.RoleKey{
 				Role:        "worker",
-				RoleHandler: role.Worker(c, providerConfig),
+				RoleHandler: p2p.Worker(c, providerConfig),
 			},
 			service.RoleKey{
 				Role:        "auto",

internal/provider/config/config.go

@@ -6,6 +6,8 @@ type Kairos struct {
 	Role         string `yaml:"role,omitempty"`
 	DNS          bool   `yaml:"dns,omitempty"`
 	LogLevel     string `yaml:"loglevel,omitempty"`
+	Hybrid       bool   `yaml:"hybrid,omitempty"`
+	MinimumNodes int    `yaml:"minimum_nodes,omitempty"`
 }
 
 type Config struct {
@@ -13,6 +15,14 @@ type Config struct {
 	K3sAgent K3s               `yaml:"k3s-agent,omitempty"`
 	K3s      K3s               `yaml:"k3s,omitempty"`
 	VPN      map[string]string `yaml:"vpn,omitempty"`
+	KubeVIP  KubeVIP           `yaml:"kubevip,omitempty"`
+}
+
+type KubeVIP struct {
+	Args        []string `yaml:"args,omitempty"`
+	EIP         string   `yaml:"eip,omitempty"`
+	ManifestURL string   `yaml:"manifest_url,omitempty"`
+	Interface   string   `yaml:"interface,omitempty"`
 }
 
 type K3s struct {

internal/provider/p2p.go

@@ -30,6 +30,52 @@ func SaveOEMCloudConfig(name string, yc yip.YipConfig) error {
 func SaveCloudConfig(name string, c []byte) error {
 	return ioutil.WriteFile(filepath.Join("oem", fmt.Sprintf("%s.yaml", name)), c, 0700)
 }
+
+func SetupAPI(apiAddress, rootDir string, start bool, c *providerConfig.Config) error {
+	if c.Kairos == nil || c.Kairos.NetworkToken == "" {
+		return fmt.Errorf("no network token defined")
+	}
+
+	svc, err := services.P2PAPI(rootDir)
+	if err != nil {
+		return fmt.Errorf("could not create svc: %w", err)
+	}
+
+	apiAddress = strings.ReplaceAll(apiAddress, "https://", "")
+	apiAddress = strings.ReplaceAll(apiAddress, "http://", "")
+
+	vpnOpts := map[string]string{
+		"EDGEVPNTOKEN": c.Kairos.NetworkToken,
+		"APILISTEN":    apiAddress,
+	}
+	// Override opts with user-supplied
+	for k, v := range c.VPN {
+		vpnOpts[k] = v
+	}
+
+	os.MkdirAll("/etc/systemd/system.conf.d/", 0600) //nolint:errcheck
+	// Setup edgevpn instance
+	err = utils.WriteEnv(filepath.Join(rootDir, "/etc/systemd/system.conf.d/edgevpn-kairos.env"), vpnOpts)
+	if err != nil {
+		return fmt.Errorf("could not create write env file: %w", err)
+	}
+
+	err = svc.WriteUnit()
+	if err != nil {
+		return fmt.Errorf("could not create write unit file: %w", err)
+	}
+
+	if start {
+		err = svc.Start()
+		if err != nil {
+			return fmt.Errorf("could not start svc: %w", err)
+		}
+
+		return svc.Enable()
+	}
+	return nil
+}
+
 func SetupVPN(instance, apiAddress, rootDir string, start bool, c *providerConfig.Config) error {
 
 	if c.Kairos == nil || c.Kairos.NetworkToken == "" {

internal/role/auto.go

@@ -17,15 +17,21 @@ func contains(slice []string, elem string) bool {
 	}
 	return false
 }
+
 func Auto(cc *config.Config, pconfig *providerConfig.Config) Role {
 	return func(c *service.RoleConfig) error {
 		advertizing, _ := c.Client.AdvertizingNodes()
 		actives, _ := c.Client.ActiveNodes()
 
+		minimumNodes := pconfig.Kairos.MinimumNodes
+		if minimumNodes == 0 {
+			minimumNodes = 2
+		}
+
 		c.Logger.Info("Active nodes:", actives)
 		c.Logger.Info("Advertizing nodes:", advertizing)
 
-		if len(advertizing) < 2 {
+		if len(advertizing) < minimumNodes {
 			c.Logger.Info("Not enough nodes")
 			return nil
 		}

internal/role/common.go

@@ -19,3 +19,16 @@ func SentinelExist() bool {
 func CreateSentinel() error {
 	return ioutil.WriteFile("/usr/local/.kairos/deployed", []byte{}, os.ModePerm)
 }
+
+func getRoles(client *service.Client, nodes []string) ([]string, map[string]string) {
+	unassignedNodes := []string{}
+	currentRoles := map[string]string{}
+	for _, a := range nodes {
+		role, _ := client.Get("role", a)
+		currentRoles[a] = role
+		if role == "" {
+			unassignedNodes = append(unassignedNodes, a)
+		}
+	}
+	return unassignedNodes, currentRoles
+}

internal/role/p2p/common.go

@@ -0,0 +1,25 @@
+package role
+
+import (
+	"fmt"
+	"net"
+
+	providerConfig "github.com/kairos-io/provider-kairos/internal/provider/config"
+)
+
+func guessInterface(pconfig *providerConfig.Config) string {
+	if pconfig.KubeVIP.Interface != "" {
+		return pconfig.KubeVIP.Interface
+	}
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		fmt.Println("failed getting system interfaces")
+		return ""
+	}
+	for _, i := range ifaces {
+		if i.Name != "lo" {
+			return i.Name
+		}
+	}
+	return ""
+}

internal/role/p2p/kubevip.go

@@ -0,0 +1,89 @@
+package role
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/kairos-io/kairos/pkg/utils"
+	"github.com/kairos-io/provider-kairos/internal/assets"
+	providerConfig "github.com/kairos-io/provider-kairos/internal/provider/config"
+)
+
+func generateKubeVIP(iface, ip string, args []string) (string, error) {
+	out, err := utils.SH(fmt.Sprintf("kube-vip manifest daemonset --interface %s --address %s --inCluster --taint --controlplane --arp --leaderElection %s", iface, ip, strings.Join(args, " ")))
+
+	if err != nil {
+		return "", fmt.Errorf("error: %w - %s", err, out)
+	}
+
+	return out, nil
+}
+
+func downloadFromURL(url, where string) error {
+	output, err := os.Create(where)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	response, err := http.Get(url)
+	if err != nil {
+		return err
+
+	}
+	defer response.Body.Close()
+
+	_, err = io.Copy(output, response.Body)
+	return err
+}
+
+func deployKubeVIP(iface, ip string, pconfig *providerConfig.Config) error {
+	if err := os.MkdirAll("/var/lib/rancher/k3s/server/manifests/", 0650); err != nil {
+		return fmt.Errorf("could not create manifest dir")
+	}
+
+	targetFile := "/var/lib/rancher/k3s/server/manifests/kubevip.yaml"
+	targetCRDFile := "/var/lib/rancher/k3s/server/manifests/kubevipmanifest.yaml"
+
+	if pconfig.KubeVIP.ManifestURL != "" {
+		err := downloadFromURL(pconfig.KubeVIP.ManifestURL, targetCRDFile)
+		if err != nil {
+			return err
+		}
+	} else {
+		f, err := assets.GetStaticFS().Open("kube_vip_rbac.yaml")
+		if err != nil {
+			return fmt.Errorf("could not find kube_vip in assets")
+		}
+		defer f.Close()
+
+		destination, err := os.Create(targetCRDFile)
+		if err != nil {
+			return err
+		}
+		defer destination.Close()
+		_, err = io.Copy(destination, f)
+		if err != nil {
+			return err
+		}
+	}
+
+	content, err := generateKubeVIP(iface, ip, pconfig.KubeVIP.Args)
+	if err != nil {
+		return fmt.Errorf("could not generate kubevip %s", err.Error())
+	}
+
+	f, err := os.Create(targetFile)
+	if err != nil {
+		return fmt.Errorf("could not open %s: %w", f.Name(), err)
+	}
+	defer f.Close()
+	if _, err := f.WriteString(content); err != nil {
+		return fmt.Errorf("could not write to %s: %w", f.Name(), err)
+	}
+
+	return nil
+}

internal/role/p2p/master.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/kairos-io/kairos/pkg/utils"
 	providerConfig "github.com/kairos-io/provider-kairos/internal/provider/config"
+	"github.com/kairos-io/provider-kairos/internal/role"
 
 	service "github.com/mudler/edgevpn/api/client/service"
 )
@@ -67,13 +68,20 @@ func propagateMasterData(ip string, c *service.RoleConfig) error {
 	return nil
 }
 
-func Master(cc *config.Config, pconfig *providerConfig.Config) Role {
+func Master(cc *config.Config, pconfig *providerConfig.Config) role.Role {
 	return func(c *service.RoleConfig) error {
 
-		ip := utils.GetInterfaceIP("edgevpn0")
+		var ip string
+		iface := guessInterface(pconfig)
+		ifaceIP := utils.GetInterfaceIP(iface)
+		if pconfig.Kairos.Hybrid {
+			ip = pconfig.KubeVIP.EIP
+		} else {
+			ip = utils.GetInterfaceIP("edgevpn0")
 			if ip == "" {
 				return errors.New("node doesn't have an ip yet")
 			}
+		}
 
 		if pconfig.Kairos.Role != "" {
 			// propagate role if we were forced by configuration
@@ -83,7 +91,7 @@ func Master(cc *config.Config, pconfig *providerConfig.Config) Role {
 			}
 		}
 
-		if SentinelExist() {
+		if role.SentinelExist() {
 			c.Logger.Info("Node already configured, backing off")
 			return propagateMasterData(ip, c)
 		}
@@ -117,7 +125,16 @@ func Master(cc *config.Config, pconfig *providerConfig.Config) Role {
 			return err
 		}
 
-		args := []string{"--flannel-iface=edgevpn0"}
+		var args []string
+		if pconfig.Kairos.Hybrid {
+			args = []string{fmt.Sprintf("--tls-san=%s", ip), fmt.Sprintf("--node-ip=%s", ifaceIP)}
+			if err := deployKubeVIP(iface, ip, pconfig); err != nil {
+				return fmt.Errorf("failed KubeVIP setup: %w", err)
+			}
+		} else {
+			args = []string{"--flannel-iface=edgevpn0"}
+		}
+
 		if k3sConfig.ReplaceArgs {
 			args = k3sConfig.Args
 		} else {
@@ -145,7 +162,7 @@ func Master(cc *config.Config, pconfig *providerConfig.Config) Role {
 			return err
 		}
 
-		return CreateSentinel()
+		return role.CreateSentinel()
 	}
 }
 

internal/role/p2p/worker.go

@@ -10,10 +10,11 @@ import (
 	"github.com/kairos-io/kairos/pkg/utils"
 
 	providerConfig "github.com/kairos-io/provider-kairos/internal/provider/config"
+	"github.com/kairos-io/provider-kairos/internal/role"
 	service "github.com/mudler/edgevpn/api/client/service"
 )
 
-func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
+func Worker(cc *config.Config, pconfig *providerConfig.Config) role.Role {
 	return func(c *service.RoleConfig) error {
 
 		if pconfig.Kairos.Role != "" {
@@ -24,7 +25,7 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
 			}
 		}
 
-		if SentinelExist() {
+		if role.SentinelExist() {
 			c.Logger.Info("Node already configured, backing off")
 			return nil
 		}
@@ -43,13 +44,6 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
 
 		nodeToken = strings.TrimRight(nodeToken, "\n")
 
-		ip := utils.GetInterfaceIP("edgevpn0")
-		if ip == "" {
-			return errors.New("node doesn't have an ip yet")
-		}
-
-		c.Logger.Info("Configuring k3s-agent", ip, masterIP, nodeToken)
-
 		svc, err := machine.K3sAgent()
 		if err != nil {
 			return err
@@ -74,6 +68,26 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
 			env = k3sConfig.Env
 		}
 
+		args := []string{
+			"--with-node-id",
+		}
+
+		if pconfig.Kairos.Hybrid {
+			iface := guessInterface(pconfig)
+			ip := utils.GetInterfaceIP(iface)
+			args = append(args,
+				fmt.Sprintf("--node-ip %s", ip))
+		} else {
+			ip := utils.GetInterfaceIP("edgevpn0")
+			if ip == "" {
+				return errors.New("node doesn't have an ip yet")
+			}
+			args = append(args,
+				fmt.Sprintf("--node-ip %s", ip),
+				"--flannel-iface=edgevpn0")
+		}
+
+		c.Logger.Info("Configuring k3s-agent", masterIP, nodeToken, args)
 		// Setup systemd unit and starts it
 		if err := utils.WriteEnv(machine.K3sEnvUnit("k3s-agent"),
 			env,
@@ -81,11 +95,6 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
 			return err
 		}
 
-		args := []string{
-			"--with-node-id",
-			fmt.Sprintf("--node-ip %s", ip),
-			"--flannel-iface=edgevpn0",
-		}
 		if k3sConfig.ReplaceArgs {
 			args = k3sConfig.Args
 		} else {
@@ -108,6 +117,6 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) Role {
 			return err
 		}
 
-		return CreateSentinel()
+		return role.CreateSentinel()
 	}
 }

internal/role/schedule.go

@@ -16,21 +16,23 @@ func scheduleRoles(nodes []string, c *service.RoleConfig, cc *config.Config, pco
 	rand.Seed(time.Now().Unix())
 
 	// Assign roles to nodes
-	currentRoles := map[string]string{}
+	unassignedNodes, currentRoles := getRoles(c.Client, nodes)
+	c.Logger.Infof("I'm the leader. My UUID is: %s.\n Current assigned roles: %+v", c.UUID, currentRoles)
 
 	existsMaster := false
-	unassignedNodes := []string{}
-	for _, a := range nodes {
-		role, _ := c.Client.Get("role", a)
-		currentRoles[a] = role
-		if role == "master" {
-			existsMaster = true
-		} else if role == "" {
-			unassignedNodes = append(unassignedNodes, a)
-		}
+
+	masterRole := "master"
+	workerRole := "worker"
+
+	if pconfig.Kairos.Hybrid {
+		c.Logger.Info("hybrid p2p with KubeVIP enabled")
 	}
 
-	c.Logger.Infof("I'm the leader. My UUID is: %s.\n Current assigned roles: %+v", c.UUID, currentRoles)
+	for _, r := range currentRoles {
+		if r == masterRole {
+			existsMaster = true
+		}
+	}
 	c.Logger.Infof("Master already present: %t", existsMaster)
 	c.Logger.Infof("Unassigned nodes: %+v", unassignedNodes)
 
@@ -55,11 +57,11 @@ func scheduleRoles(nodes []string, c *service.RoleConfig, cc *config.Config, pco
 			selected = toSelect[rand.Intn(len(toSelect)-1)]
 		}
 
-		if err := c.Client.Set("role", selected, "master"); err != nil {
+		if err := c.Client.Set("role", selected, masterRole); err != nil {
 			return err
 		}
 		c.Logger.Info("-> Set master to", selected)
-		currentRoles[selected] = "master"
+		currentRoles[selected] = masterRole
 		// Return here, so next time we get called
 		// makes sure master is set.
 		return nil
@@ -67,11 +69,11 @@ func scheduleRoles(nodes []string, c *service.RoleConfig, cc *config.Config, pco
 
 	// cycle all empty roles and assign worker roles
 	for _, uuid := range unassignedNodes {
-		if err := c.Client.Set("role", uuid, "worker"); err != nil {
+		if err := c.Client.Set("role", uuid, workerRole); err != nil {
 			c.Logger.Error(err)
 			return err
 		}
-		c.Logger.Info("-> Set worker to", uuid)
+		c.Logger.Infof("-> Set %s to %s", workerRole, uuid)
 	}
 
 	c.Logger.Info("Done scheduling")

internal/services/edgevpn.go

@@ -26,6 +26,35 @@ if [ -f /etc/environment ]; then source /etc/environment; fi
 if [ -f /etc/systemd/system.conf.d/edgevpn-kairos.env ]; then source /etc/systemd/system.conf.d/edgevpn-kairos.env; fi
 set +o allexport`
 
+const edgevpnAPIOpenRC string = `#!/sbin/openrc-run
+
+depend() {
+	after net
+	provide edgevpn
+}
+
+supervisor=supervise-daemon
+name="edgevpn"
+command="edgevpn api --enable-healthchecks"
+supervise_daemon_args="--stdout /var/log/edgevpn.log --stderr /var/log/edgevpn.log"
+pidfile="/run/edgevpn.pid"
+respawn_delay=5
+set -o allexport
+if [ -f /etc/environment ]; then source /etc/environment; fi
+if [ -f /etc/systemd/system.conf.d/edgevpn-kairos.env ]; then source /etc/systemd/system.conf.d/edgevpn-kairos.env; fi
+set +o allexport`
+
+const edgevpnAPISystemd string = `[Unit]
+Description=P2P API Daemon
+After=network.target
+[Service]
+EnvironmentFile=/etc/systemd/system.conf.d/edgevpn-kairos.env
+LimitNOFILE=49152
+ExecStart=edgevpn api --enable-healthchecks
+Restart=always
+[Install]
+WantedBy=multi-user.target`
+
 const edgevpnSystemd string = `[Unit]
 Description=EdgeVPN Daemon
 After=network.target
@@ -55,3 +84,19 @@ func EdgeVPN(instance, rootDir string) (machine.Service, error) {
 		systemd.WithRoot(rootDir),
 	)
 }
+
+func P2PAPI(rootDir string) (machine.Service, error) {
+	if utils.IsOpenRCBased() {
+		return openrc.NewService(
+			openrc.WithName("edgevpn"),
+			openrc.WithUnitContent(edgevpnAPIOpenRC),
+			openrc.WithRoot(rootDir),
+		)
+	}
+
+	return systemd.NewService(
+		systemd.WithName("edgevpn"),
+		systemd.WithUnitContent(edgevpnAPISystemd),
+		systemd.WithRoot(rootDir),
+	)
+}

repository.yaml

@@ -9,9 +9,9 @@ repositories:
     priority: 2
     urls:
       - "quay.io/kairos/packages"
-    reference: 20221125103743-repository.yaml
+    reference: 20221202173038-repository.yaml
   - !!merge <<: *kairos
     arch: arm64
     urls:
       - "quay.io/kairos/packages-arm64"
-    reference: 20221125104907-repository.yaml
+    reference: 20221202173723-repository.yaml