Go back to having a k8sNode

Signed-off-by: Mauro Morales <mauro.morales@spectrocloud.com>
This commit is contained in:
Mauro Morales
2025-03-26 15:15:40 +01:00
parent 65e49b898b
commit e987847a7a
6 changed files with 502 additions and 589 deletions

View File

@@ -43,15 +43,15 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
if err != nil {
return ErrorEvent("Failed reading JSON input: %s input '%s'", err.Error(), cfg.Config)
}
// TODO: this belong to a systemd service that is started instead
p2pBlockDefined := prvConfig.P2P != nil
tokenNotDefined := (p2pBlockDefined && prvConfig.P2P.NetworkToken == "") || !p2pBlockDefined
skipAuto := p2pBlockDefined && !prvConfig.P2P.Auto.IsEnabled()
sd, _ := p2p.NewServiceDefinition(prvConfig)
if prvConfig.P2P == nil && sd == nil {
return pluggable.EventResponse{State: fmt.Sprintf("no kubernetes distribution configuration. nothing to do: %s", cfg.Config)}
// Try to create a node first - this validates we can actually create a working node
_, err = p2p.NewNode(prvConfig, common.RoleAuto)
if err != nil {
return ErrorEvent("Cannot create Kubernetes node: %s", err.Error())
}
utils.SH("kairos-agent run-stage kairos-agent.bootstrap") //nolint:errcheck
@@ -65,10 +65,8 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
logger := types.NewKairosLogger("provider", logLevel, false)
// Do onetimebootstrap if a Kubernetes distribution is enabled.
// Those blocks are not required to be enabled in case of a kairos
// full automated setup. Otherwise, they must be explicitly enabled.
if (tokenNotDefined && sd != nil) || skipAuto {
// Do onetimebootstrap if needed - we already validated we can create a node
if tokenNotDefined || skipAuto {
err := oneTimeBootstrap(logger, prvConfig, func() error {
return SetupVPN(services.EdgeVPNDefaultInstance, cfg.APIAddress, "/", true, prvConfig)
})
@@ -79,7 +77,7 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
}
if tokenNotDefined {
return ErrorEvent("No network token provided, or kubernetes distribution (k3s, k0s) block configured. Exiting")
return ErrorEvent("No network token provided. Exiting")
}
// We might still want a VPN, but not to route traffic into
@@ -135,7 +133,7 @@ func Bootstrap(e *pluggable.Event) pluggable.EventResponse {
RoleHandler: role.Auto(c, prvConfig),
},
service.RoleKey{
Role: common.RoleControlPlane,
Role: common.RoleMaster,
RoleHandler: p2p.ControlPlane(c, prvConfig, common.RoleControlPlane),
},
service.RoleKey{
@@ -179,27 +177,28 @@ func oneTimeBootstrap(l types.KairosLogger, c *providerConfig.Config, vpnSetupFN
l.Info("One time bootstrap starting")
var svc machine.Service
sd, err := p2p.NewServiceDefinition(c)
node, err := p2p.NewNode(c, common.RoleAuto)
if err != nil {
l.Info("No Kubernetes configuration found, skipping bootstrap.")
return nil
}
if sd.K8sBin() == "" {
l.Errorf("no %s binary fouund", sd.ServiceName())
return fmt.Errorf("no %s binary found", sd.ServiceName())
k8sBin := node.K8sBin()
if k8sBin == "" {
l.Errorf("no kubernetes binary found")
return fmt.Errorf("no kubernetes binary found")
}
if err := utils.WriteEnv(sd.EnvFile(), sd.Env()); err != nil {
l.Errorf("Failed to write %s env file: %s", sd.ServiceName(), err.Error())
if err := utils.WriteEnv(node.GetEnvFile(), node.GenerateEnv()); err != nil {
l.Errorf("Failed to write env file: %s", err.Error())
return err
}
// Initialize the service based on the system's init system
if utils.IsOpenRCBased() {
svc, err = openrc.NewService(openrc.WithName(sd.ServiceName()))
svc, err = openrc.NewService(openrc.WithName(node.GetServiceName()))
} else {
svc, err = systemd.NewService(systemd.WithName(sd.ServiceName()))
svc, err = systemd.NewService(systemd.WithName(node.GetServiceName()))
}
if err != nil {
@@ -211,12 +210,12 @@ func oneTimeBootstrap(l types.KairosLogger, c *providerConfig.Config, vpnSetupFN
}
// Override the service command and start it
args, err := sd.Args()
args, err := node.GenerateArgs()
if err != nil {
l.Errorf("Failed to generate %s args: %s", sd.ServiceName(), err.Error())
l.Errorf("Failed to generate args: %s", err.Error())
return err
}
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", sd.K8sBin(), sd.Role(), strings.Join(args, " "))); err != nil {
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", k8sBin, node.GetRole(), strings.Join(args, " "))); err != nil {
l.Errorf("Failed to override service command: %s", err.Error())
return err
}

View File

@@ -14,8 +14,8 @@ import (
service "github.com/mudler/edgevpn/api/client/service"
)
func propagateControlPlaneData(role string, k K8sControlPlane) error {
c := k.RoleConfig()
func propagateControlPlaneData(role string, k ControlPlaneNode) error {
c := k.GetRoleConfig()
defer func() {
// Avoid polluting the API.
// The ledger already retries in the background to update the blockchain, but it has
@@ -31,7 +31,7 @@ func propagateControlPlaneData(role string, k K8sControlPlane) error {
return err
}
if k.HA() && !k.ClusterInit() {
if k.IsHA() && !k.IsClusterInit() {
return nil
}
@@ -40,7 +40,7 @@ func propagateControlPlaneData(role string, k K8sControlPlane) error {
c.Logger.Error(err)
}
err = c.Client.Set("control-plane", "ip", k.IP())
err = c.Client.Set("control-plane", "ip", k.GetIP())
if err != nil {
c.Logger.Error(err)
}
@@ -55,11 +55,11 @@ func guessIP(pconfig *providerConfig.Config) string {
return utils.GetInterfaceIP("edgevpn0")
}
func waitForControlPlaneHAInfo(m K8sControlPlane) bool {
func waitForControlPlaneHAInfo(m ControlPlaneNode) bool {
var controlPlaneToken string
controlPlaneToken, _ = m.Token()
c := m.RoleConfig()
controlPlaneToken, _ = m.GetToken()
c := m.GetRoleConfig()
if controlPlaneToken == "" {
c.Logger.Info("Control Plane's token is not there yet..")
@@ -99,15 +99,22 @@ func ControlPlane(cc *config.Config, pconfig *providerConfig.Config, roleName st
}
c.Logger.Info("Determining K8s distro")
controlPlane, err := NewK8sControlPlane(pconfig)
node, err := NewNode(pconfig, roleName)
if err != nil {
return fmt.Errorf("failed to determine k8s distro: %w", err)
}
controlPlane, ok := AsControlPlane(node)
if !ok {
return fmt.Errorf("failed to convert node to control plane")
}
controlPlane.SetRole(roleName)
controlPlane.SetRoleConfig(c)
controlPlane.SetIP(ip)
controlPlane.GuessInterface()
if k3sNode, ok := node.(*K3sNode); ok {
k3sNode.GuessInterface()
}
c.Logger.Info("Verifying sentinel file")
if role.SentinelExist() {
@@ -116,7 +123,7 @@ func ControlPlane(cc *config.Config, pconfig *providerConfig.Config, roleName st
}
c.Logger.Info("Checking HA")
if controlPlane.HA() && !controlPlane.ClusterInit() && waitForControlPlaneHAInfo(controlPlane) {
if controlPlane.IsHA() && !controlPlane.IsClusterInit() && waitForControlPlaneHAInfo(controlPlane) {
return nil
}
@@ -124,38 +131,38 @@ func ControlPlane(cc *config.Config, pconfig *providerConfig.Config, roleName st
env := controlPlane.GenerateEnv()
// Configure k8s service to start on edgevpn0
c.Logger.Info(fmt.Sprintf("Configuring %s", controlPlane.Distro()))
c.Logger.Info(fmt.Sprintf("Configuring %s", controlPlane.GetDistro()))
c.Logger.Info("Running bootstrap before stage")
utils.SH(fmt.Sprintf("kairos-agent run-stage provider-kairos.bootstrap.before.%s", roleName)) //nolint:errcheck
if controlPlane.HA() {
if controlPlane.IsHA() {
err = controlPlane.SetupHAToken()
if err != nil {
return err
}
}
svc, err := controlPlane.Service()
svc, err := controlPlane.GetService()
if err != nil {
return fmt.Errorf("failed to get %s service: %w", controlPlane.Distro(), err)
return fmt.Errorf("failed to get %s service: %w", controlPlane.GetDistro(), err)
}
c.Logger.Info("Writing service Env %s")
envUnit := controlPlane.EnvUnit()
envUnit := controlPlane.GetEnvFile()
if err := utils.WriteEnv(envUnit,
env,
); err != nil {
return fmt.Errorf("failed to write the %s service: %w", controlPlane.Distro(), err)
return fmt.Errorf("failed to write the %s service: %w", controlPlane.GetDistro(), err)
}
c.Logger.Info("Generating args")
args, err := controlPlane.Args()
args, err := controlPlane.GenerateArgs()
if err != nil {
return fmt.Errorf("failed to generate %s args: %w", controlPlane.Distro(), err)
return fmt.Errorf("failed to generate %s args: %w", controlPlane.GetDistro(), err)
}
if controlPlane.ProviderConfig().KubeVIP.IsEnabled() {
if controlPlane.GetConfig().KubeVIP.IsEnabled() {
c.Logger.Info("Configuring KubeVIP")
if err := controlPlane.DeployKubeVIP(); err != nil {
return fmt.Errorf("failed KubeVIP setup: %w", err)
@@ -164,22 +171,22 @@ func ControlPlane(cc *config.Config, pconfig *providerConfig.Config, roleName st
k8sBin := controlPlane.K8sBin()
if k8sBin == "" {
return fmt.Errorf("no %s binary found (?)", controlPlane.Distro())
return fmt.Errorf("no %s binary found (?)", controlPlane.GetDistro())
}
c.Logger.Info("Writing service override")
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", k8sBin, controlPlane.Role(), strings.Join(args, " "))); err != nil {
return fmt.Errorf("failed to override %s command: %w", controlPlane.Distro(), err)
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", k8sBin, controlPlane.GetRole(), strings.Join(args, " "))); err != nil {
return fmt.Errorf("failed to override %s command: %w", controlPlane.GetDistro(), err)
}
c.Logger.Info("Starting service")
if err := svc.Start(); err != nil {
return fmt.Errorf("failed to start %s service: %w", controlPlane.Distro(), err)
return fmt.Errorf("failed to start %s service: %w", controlPlane.GetDistro(), err)
}
c.Logger.Info("Enabling service")
if err := svc.Enable(); err != nil {
return fmt.Errorf("failed to enable %s service: %w", controlPlane.Distro(), err)
return fmt.Errorf("failed to enable %s service: %w", controlPlane.GetDistro(), err)
}
c.Logger.Info("Propagating control plane data")

View File

@@ -18,30 +18,152 @@ const (
K0sDistroName = "k0s"
)
// K0sNode implements the base Node interface for K0s
type K0sNode struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
role string
}
// K0sControlPlane extends K0sNode with control plane functionality
type K0sControlPlane struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
role string
*K0sNode
}
// K0sWorker extends K0sNode with worker functionality
type K0sWorker struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
role string
*K0sNode
}
func (k *K0sControlPlane) K8sBin() string {
// Node interface implementation
func (k *K0sNode) GetIP() string {
return k.ip
}
func (k *K0sNode) SetIP(ip string) {
k.ip = ip
}
func (k *K0sNode) GetRole() string {
return k.role
}
func (k *K0sNode) SetRole(role string) {
k.role = role
}
func (k *K0sNode) GetDistro() string {
return K0sDistroName
}
func (k *K0sNode) K8sBin() string {
return utils.K0sBin()
}
func (k *K0sWorker) K8sBin() string {
return utils.K0sBin()
func (k *K0sNode) GetConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K0sNode) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K0sNode) GetRoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K0sNode) GetService() (machine.Service, error) {
if k.role == common.RoleWorker {
return machine.K0sWorker()
}
return machine.K0s()
}
func (k *K0sNode) GetServiceName() string {
if k.role == common.RoleWorker {
return "k0sworker"
}
return "k0scontroller"
}
func (k *K0sNode) GetEnvFile() string {
return machine.K0sEnvUnit(k.GetServiceName())
}
func (k *K0sNode) GenerateEnv() map[string]string {
env := make(map[string]string)
if k.role == common.RoleControlPlaneHA && k.role != common.RoleControlPlaneClusterInit {
nodeToken, _ := k.GetToken()
env["K0S_TOKEN"] = nodeToken
}
pConfig := k.GetConfig()
if k.role == common.RoleWorker {
if pConfig.K0sWorker.ReplaceEnv {
env = pConfig.K0sWorker.Env
} else {
for k, v := range pConfig.K0sWorker.Env {
env[k] = v
}
}
} else {
if pConfig.K0s.ReplaceEnv {
env = pConfig.K0s.Env
} else {
for k, v := range pConfig.K0s.Env {
env[k] = v
}
}
}
return env
}
func (k *K0sNode) GenerateArgs() ([]string, error) {
if k.role == common.RoleWorker {
return k.generateWorkerArgs()
}
return k.generateControlPlaneArgs()
}
func (k *K0sNode) GetToken() (string, error) {
if k.role == common.RoleWorker {
return k.GetRoleConfig().Client.Get("workertoken", "token")
}
return k.GetRoleConfig().Client.Get("controllertoken", "token")
}
// ControlPlaneNode interface implementation
func (k *K0sControlPlane) IsHA() bool {
return k.role == common.RoleControlPlaneHA
}
func (k *K0sControlPlane) IsClusterInit() bool {
return k.role == common.RoleControlPlaneClusterInit
}
func (k *K0sControlPlane) SetupHAToken() error {
controlPlaneToken, err := k.GetToken()
if err != nil {
return err
}
if controlPlaneToken == "" {
return errors.New("control plane token is not there")
}
if err := os.WriteFile("/etc/k0s/token", []byte(controlPlaneToken), 0644); err != nil {
return err
}
return nil
}
func (k *K0sControlPlane) DeployKubeVIP() error {
pconfig := k.ProviderConfig()
pconfig := k.GetConfig()
if pconfig.KubeVIP.IsEnabled() {
return errors.New("KubeVIP is not yet supported with k0s")
}
@@ -49,12 +171,18 @@ func (k *K0sControlPlane) DeployKubeVIP() error {
return nil
}
func (k *K0sControlPlane) Args() ([]string, error) {
var args []string
// WorkerNode interface implementation
func (k *K0sWorker) SetupWorker(_, nodeToken string) error {
if err := os.WriteFile("/etc/k0s/token", []byte(nodeToken), 0644); err != nil {
return err
}
// if k.IsSingleNode() {
// args = append(args, "--single")
// }
return nil
}
// Helper methods
func (k *K0sNode) generateControlPlaneArgs() ([]string, error) {
var args []string
// Generate a new k0s config
_, err := utils.SH("k0s config create > /etc/k0s/k0s.yaml")
@@ -84,7 +212,7 @@ func (k *K0sControlPlane) Args() ([]string, error) {
return args, errors.New("k0s config does not have an api")
}
// by default k0s uses the first IP address of the machine as the api address, but we want to use the edgevpn IP
api["address"] = k.IP()
api["address"] = k.GetIP()
spec["api"] = api
@@ -111,7 +239,7 @@ func (k *K0sControlPlane) Args() ([]string, error) {
return args, errors.New("k0s config does not have a etcd")
}
// just like the api address, we want to use the edgevpn IP for the etcd peer address
etcd["peerAddress"] = k.IP()
etcd["peerAddress"] = k.GetIP()
storage["etcd"] = etcd
spec["storage"] = storage
@@ -128,7 +256,7 @@ func (k *K0sControlPlane) Args() ([]string, error) {
return args, err
}
pconfig := k.ProviderConfig()
pconfig := k.GetConfig()
if !pconfig.P2P.UseVPNWithKubernetes() {
return args, errors.New("Having a VPN but not using it for Kubernetes is not yet supported with k0s")
}
@@ -141,104 +269,27 @@ func (k *K0sControlPlane) Args() ([]string, error) {
return args, errors.New("ExternalDB is not yet supported with k0s")
}
if k.HA() {
if k.role == common.RoleControlPlaneHA {
args = append(args, "--token-file /etc/k0s/token")
}
// when we start implementing this functionality, remember to use
// AppendArgs, and not just return the args here, this is because the
// function understands if it needs to append or replace the args
return args, nil
}
func (k *K0sControlPlane) EnvUnit() string {
return machine.K0sEnvUnit("k0scontroller")
}
func (k *K0sNode) generateWorkerArgs() ([]string, error) {
pconfig := k.GetConfig()
k0sConfig := pconfig.K0sWorker
args := []string{"--token-file /etc/k0s/token"}
func (k *K0sControlPlane) Service() (machine.Service, error) {
return machine.K0s()
}
func (k *K0sWorker) Service() (machine.Service, error) {
return machine.K0sWorker()
}
func (k *K0sControlPlane) Token() (string, error) {
return k.RoleConfig().Client.Get("controllertoken", "token")
}
func (k *K0sWorker) Token() (string, error) {
return k.RoleConfig().Client.Get("workertoken", "token")
}
func (k *K0sControlPlane) GenerateEnv() (env map[string]string) {
env = make(map[string]string)
if k.HA() && !k.ClusterInit() {
nodeToken, _ := k.Token()
env["K0S_TOKEN"] = nodeToken
if k0sConfig.ReplaceArgs {
return k0sConfig.Args, nil
}
pConfig := k.ProviderConfig()
if pConfig.K0s.ReplaceEnv {
env = pConfig.K0s.Env
} else {
// Override opts with user-supplied
for k, v := range pConfig.K0s.Env {
env[k] = v
}
}
return env
return append(args, k0sConfig.Args...), nil
}
func (k *K0sControlPlane) ProviderConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K0sWorker) ProviderConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K0sControlPlane) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K0sWorker) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K0sControlPlane) RoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K0sWorker) RoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K0sControlPlane) IsSingleNode() bool {
return k.role == common.RoleControlPlane
}
func (k *K0sControlPlane) HA() bool {
return k.role == common.RoleControlPlaneHA
}
func (k *K0sControlPlane) ClusterInit() bool {
return k.role == common.RoleControlPlaneClusterInit
}
func (k *K0sControlPlane) IP() string {
return k.ip
}
func (k *K0sWorker) IP() string {
return k.ip
}
func (k *K0sControlPlane) PropagateData() error {
c := k.RoleConfig()
func (k *K0sNode) PropagateData() error {
c := k.GetRoleConfig()
controllerToken, err := utils.SH("k0s token create --role=controller") //nolint:errcheck
if err != nil {
c.Logger.Errorf("failed to create controller token: %s", err)
@@ -278,104 +329,3 @@ func (k *K0sControlPlane) PropagateData() error {
return nil
}
func (k *K0sWorker) Args() ([]string, error) {
pconfig := k.ProviderConfig()
k0sConfig := pconfig.K0sWorker
args := []string{"--token-file /etc/k0s/token"}
if k0sConfig.ReplaceArgs {
args = k0sConfig.Args
} else {
args = append(args, k0sConfig.Args...)
}
return args, nil
}
func (k *K0sControlPlane) SetupHAToken() error {
controlPlaneToken, err := k.Token()
if err != nil {
return err
}
if controlPlaneToken == "" {
return errors.New("control plane token is not there")
}
if err := os.WriteFile("/etc/k0s/token", []byte(controlPlaneToken), 0644); err != nil {
return err
}
return nil
}
func (k *K0sWorker) SetupWorker(_, nodeToken string) error {
if err := os.WriteFile("/etc/k0s/token", []byte(nodeToken), 0644); err != nil {
return err
}
return nil
}
func (k *K0sControlPlane) Role() string {
return "controller"
}
func (k *K0sWorker) Role() string {
return "worker"
}
func (k *K0sControlPlane) ServiceName() string {
return "k0scontroller"
}
func (k *K0sWorker) ServiceName() string {
return "k0sworker"
}
func (k *K0sControlPlane) Env() map[string]string {
c := k.ProviderConfig()
return c.K0s.Env
}
func (k *K0sWorker) Env() map[string]string {
c := k.ProviderConfig()
return c.K0sWorker.Env
}
func (k *K0sControlPlane) EnvFile() string {
return machine.K0sEnvUnit(k.ServiceName())
}
func (k *K0sWorker) EnvFile() string {
return machine.K0sEnvUnit(k.ServiceName())
}
func (k *K0sControlPlane) SetRole(role string) {
k.role = role
}
func (k *K0sWorker) SetRole(role string) {
k.role = role
}
func (k *K0sControlPlane) SetIP(ip string) {
k.ip = ip
}
func (k *K0sWorker) SetIP(ip string) {
k.ip = ip
}
func (k *K0sControlPlane) GuessInterface() {
// not used in k0s
}
func (k *K0sControlPlane) Distro() string {
return K0sDistroName
}
func (k *K0sWorker) Distro() string {
return K0sDistroName
}

View File

@@ -18,34 +18,150 @@ const (
K3sDistroName = "k3s"
)
// K3sNode implements the base Node interface for K3s
type K3sNode struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
iface string
ifaceIP string
role string
}
// K3sControlPlane extends K3sNode with control plane functionality
type K3sControlPlane struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
iface string
ifaceIP string
role string
*K3sNode
}
// K3sWorker extends K3sNode with worker functionality
type K3sWorker struct {
providerConfig *providerConfig.Config
roleConfig *service.RoleConfig
ip string
iface string
ifaceIP string
role string
*K3sNode
}
func (k *K3sControlPlane) K8sBin() string {
// Node interface implementation
func (k *K3sNode) GetIP() string {
return k.ip
}
func (k *K3sNode) SetIP(ip string) {
k.ip = ip
}
func (k *K3sNode) GetRole() string {
if k.role == common.RoleControlPlane ||
k.role == common.RoleControlPlaneHA ||
k.role == common.RoleControlPlaneClusterInit ||
k.role == common.RoleMaster ||
k.role == common.RoleMasterHA ||
k.role == common.RoleMasterInit {
return "server"
}
return "agent"
}
func (k *K3sNode) SetRole(role string) {
k.role = role
}
func (k *K3sNode) GetDistro() string {
return K3sDistroName
}
func (k *K3sNode) K8sBin() string {
return utils.K3sBin()
}
func (k *K3sWorker) K8sBin() string {
return utils.K3sBin()
func (k *K3sNode) GetConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K3sNode) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K3sNode) GetRoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K3sNode) GetService() (machine.Service, error) {
if k.role == common.RoleWorker {
return machine.K3sAgent()
}
return machine.K3s()
}
func (k *K3sNode) GetServiceName() string {
if k.role == common.RoleWorker {
return "k3s-agent"
}
return "k3s"
}
func (k *K3sNode) GetEnvFile() string {
return machine.K3sEnvUnit(k.GetServiceName())
}
func (k *K3sNode) GenerateEnv() map[string]string {
env := make(map[string]string)
if k.role == common.RoleControlPlaneHA && k.role != common.RoleControlPlaneClusterInit {
nodeToken, _ := k.GetToken()
env["K3S_TOKEN"] = nodeToken
}
pConfig := k.GetConfig()
if k.role == common.RoleWorker {
if pConfig.K3sAgent.ReplaceEnv {
env = pConfig.K3sAgent.Env
} else {
for k, v := range pConfig.K3sAgent.Env {
env[k] = v
}
}
} else {
if pConfig.K3s.ReplaceEnv {
env = pConfig.K3s.Env
} else {
for k, v := range pConfig.K3s.Env {
env[k] = v
}
}
}
return env
}
func (k *K3sNode) GenerateArgs() ([]string, error) {
if k.role == common.RoleWorker {
return k.generateWorkerArgs()
}
return k.generateControlPlaneArgs()
}
func (k *K3sNode) GetToken() (string, error) {
if k.role == common.RoleWorker {
return k.GetRoleConfig().Client.Get("nodetoken", "token")
}
return k.GetRoleConfig().Client.Get("nodetoken", "token")
}
// ControlPlane interface implementation
func (k *K3sControlPlane) IsHA() bool {
return k.role == common.RoleControlPlaneHA
}
func (k *K3sControlPlane) IsClusterInit() bool {
return k.role == common.RoleControlPlaneClusterInit
}
func (k *K3sControlPlane) SetupHAToken() error {
// K3s doesn't need a token for HA, it uses the node-token
return nil
}
func (k *K3sControlPlane) DeployKubeVIP() error {
pconfig := k.ProviderConfig()
pconfig := k.GetConfig()
if !pconfig.KubeVIP.IsEnabled() {
return nil
}
@@ -53,9 +169,43 @@ func (k *K3sControlPlane) DeployKubeVIP() error {
return deployKubeVIP(k.iface, k.ip, pconfig)
}
func (k *K3sControlPlane) Args() ([]string, error) {
// Worker interface implementation
func (k *K3sWorker) SetupWorker(controlPlaneIP, nodeToken string) error {
pconfig := k.GetConfig()
nodeToken = strings.TrimRight(nodeToken, "\n")
k3sConfig := providerConfig.K3s{}
if pconfig.K3sAgent.Enabled {
k3sConfig = pconfig.K3sAgent
}
env := map[string]string{
"K3S_URL": fmt.Sprintf("https://%s:6443", controlPlaneIP),
"K3S_TOKEN": nodeToken,
}
if k3sConfig.ReplaceEnv {
env = k3sConfig.Env
} else {
for k, v := range k3sConfig.Env {
env[k] = v
}
}
if err := utils.WriteEnv(machine.K3sEnvUnit("k3s-agent"),
env,
); err != nil {
return err
}
return nil
}
// Helper methods
func (k *K3sNode) generateControlPlaneArgs() ([]string, error) {
var args []string
pconfig := k.ProviderConfig()
pconfig := k.GetConfig()
if pconfig.P2P.UseVPNWithKubernetes() {
args = append(args, "--flannel-iface=edgevpn0")
@@ -73,127 +223,58 @@ func (k *K3sControlPlane) Args() ([]string, error) {
args = []string{fmt.Sprintf("--datastore-endpoint=%s", pconfig.P2P.Auto.HA.ExternalDB)}
}
if k.HA() && !k.ClusterInit() {
clusterInitIP, _ := k.roleConfig.Client.Get("control-plane", "ip")
if k.role == common.RoleControlPlaneHA && k.role != common.RoleControlPlaneClusterInit {
clusterInitIP, _ := k.GetRoleConfig().Client.Get("control-plane", "ip")
args = append(args, fmt.Sprintf("--server=https://%s:6443", clusterInitIP))
}
// The --cluster-init flag changes the embedded SQLite DB to etcd. We don't
// want to do this if we're using an external DB.
if k.ClusterInit() && pconfig.P2P.Auto.HA.ExternalDB == "" {
if k.role == common.RoleControlPlaneClusterInit && pconfig.P2P.Auto.HA.ExternalDB == "" {
args = append(args, "--cluster-init")
}
args = k.AppendArgs(args)
return args, nil
}
func (k *K3sControlPlane) AppendArgs(other []string) []string {
c := k.ProviderConfig()
if c.K3s.ReplaceArgs {
return c.K3s.Args
if pconfig.K3s.ReplaceArgs {
return pconfig.K3s.Args, nil
}
return append(other, c.K3s.Args...)
return append(args, pconfig.K3s.Args...), nil
}
func (k *K3sWorker) AppendArgs(other []string) []string {
c := k.ProviderConfig()
if c.K3s.ReplaceArgs {
return c.K3s.Args
func (k *K3sNode) generateWorkerArgs() ([]string, error) {
pconfig := k.GetConfig()
k3sConfig := providerConfig.K3s{}
if pconfig.K3sAgent.Enabled {
k3sConfig = pconfig.K3sAgent
}
return append(other, c.K3s.Args...)
}
func (k *K3sControlPlane) EnvUnit() string {
return machine.K3sEnvUnit("k3s")
}
func (k *K3sWorker) EnvUnit() string {
return machine.K3sEnvUnit("k3s")
}
func (k *K3sControlPlane) Service() (machine.Service, error) {
return machine.K3s()
}
func (k *K3sWorker) Service() (machine.Service, error) {
return machine.K3sAgent()
}
func (k *K3sControlPlane) Token() (string, error) {
return k.RoleConfig().Client.Get("nodetoken", "token")
}
func (k *K3sWorker) Token() (string, error) {
return k.RoleConfig().Client.Get("nodetoken", "token")
}
func (k *K3sControlPlane) GenerateEnv() (env map[string]string) {
env = make(map[string]string)
if k.HA() && !k.ClusterInit() {
nodeToken, _ := k.Token()
env["K3S_TOKEN"] = nodeToken
args := []string{
"--with-node-id",
}
pConfig := k.ProviderConfig()
if pConfig.K3s.ReplaceEnv {
env = pConfig.K3s.Env
} else {
// Override opts with user-supplied
for k, v := range pConfig.K3s.Env {
env[k] = v
if pconfig.P2P.UseVPNWithKubernetes() {
ip := utils.GetInterfaceIP("edgevpn0")
if ip == "" {
return nil, errors.New("node doesn't have an ip yet")
}
args = append(args,
fmt.Sprintf("--node-ip %s", ip),
"--flannel-iface=edgevpn0")
} else {
iface := guessInterface(pconfig)
ip := utils.GetInterfaceIP(iface)
args = append(args,
fmt.Sprintf("--node-ip %s", ip))
}
return env
if k3sConfig.ReplaceArgs {
return k3sConfig.Args, nil
}
return append(args, k3sConfig.Args...), nil
}
func (k *K3sControlPlane) ProviderConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K3sWorker) ProviderConfig() *providerConfig.Config {
return k.providerConfig
}
func (k *K3sControlPlane) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K3sWorker) SetRoleConfig(c *service.RoleConfig) {
k.roleConfig = c
}
func (k *K3sControlPlane) RoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K3sWorker) RoleConfig() *service.RoleConfig {
return k.roleConfig
}
func (k *K3sControlPlane) HA() bool {
return k.role == common.RoleControlPlaneHA
}
func (k *K3sControlPlane) ClusterInit() bool {
return k.role == common.RoleControlPlaneClusterInit
}
func (k *K3sControlPlane) IP() string {
return k.ip
}
func (k *K3sWorker) IP() string {
return k.ip
}
func (k *K3sControlPlane) PropagateData() error {
c := k.RoleConfig()
func (k *K3sNode) PropagateData() error {
c := k.GetRoleConfig()
tokenB, err := os.ReadFile("/var/lib/rancher/k3s/server/node-token")
if err != nil {
c.Logger.Error(err)
@@ -225,142 +306,22 @@ func (k *K3sControlPlane) PropagateData() error {
return nil
}
func (k *K3sWorker) Args() ([]string, error) {
pconfig := k.ProviderConfig()
k3sConfig := providerConfig.K3s{}
if pconfig.K3sAgent.Enabled {
k3sConfig = pconfig.K3sAgent
}
args := []string{
"--with-node-id",
}
if pconfig.P2P.UseVPNWithKubernetes() {
ip := utils.GetInterfaceIP("edgevpn0")
if ip == "" {
return nil, errors.New("node doesn't have an ip yet")
}
args = append(args,
fmt.Sprintf("--node-ip %s", ip),
"--flannel-iface=edgevpn0")
} else {
iface := guessInterface(pconfig)
ip := utils.GetInterfaceIP(iface)
args = append(args,
fmt.Sprintf("--node-ip %s", ip))
}
if k3sConfig.ReplaceArgs {
args = k3sConfig.Args
} else {
args = append(args, k3sConfig.Args...)
}
return args, nil
}
func (k *K3sControlPlane) SetupHAToken() error {
// K3s doesn't need a token for HA, it uses the node-token
return nil
}
func (k *K3sWorker) SetupWorker(controlPlaneIP, nodeToken string) error {
pconfig := k.ProviderConfig()
nodeToken = strings.TrimRight(nodeToken, "\n")
k3sConfig := providerConfig.K3s{}
if pconfig.K3sAgent.Enabled {
k3sConfig = pconfig.K3sAgent
}
env := map[string]string{
"K3S_URL": fmt.Sprintf("https://%s:6443", controlPlaneIP),
"K3S_TOKEN": nodeToken,
}
if k3sConfig.ReplaceEnv {
env = k3sConfig.Env
} else {
// Override opts with user-supplied
for k, v := range k3sConfig.Env {
env[k] = v
}
}
if err := utils.WriteEnv(machine.K3sEnvUnit("k3s-agent"),
env,
); err != nil {
return err
}
return nil
}
func (k *K3sControlPlane) Role() string {
return "server"
}
func (k *K3sWorker) Role() string {
return "agent"
}
func (k *K3sControlPlane) ServiceName() string {
return "k3s"
}
func (k *K3sWorker) ServiceName() string {
return "k3s-agent"
}
func (k *K3sControlPlane) Env() map[string]string {
c := k.ProviderConfig()
return c.K3s.Env
}
func (k *K3sWorker) Env() map[string]string {
c := k.ProviderConfig()
return c.K3sAgent.Env
}
func (k *K3sControlPlane) EnvFile() string {
return machine.K3sEnvUnit(k.ServiceName())
}
func (k *K3sWorker) EnvFile() string {
return machine.K3sEnvUnit(k.ServiceName())
}
func (k *K3sControlPlane) SetRole(role string) {
k.role = role
}
func (k *K3sWorker) SetRole(role string) {
k.role = role
}
func (k *K3sControlPlane) SetIP(ip string) {
k.ip = ip
}
func (k *K3sWorker) SetIP(ip string) {
k.ip = ip
}
func (k *K3sControlPlane) GuessInterface() {
iface := guessInterface(k.ProviderConfig())
func (k *K3sNode) GuessInterface() {
iface := guessInterface(k.GetConfig())
ifaceIP := utils.GetInterfaceIP(iface)
k.iface = iface
k.ifaceIP = ifaceIP
}
func (k *K3sControlPlane) Distro() string {
return K3sDistroName
}
func (k *K3sWorker) Distro() string {
return K3sDistroName
func (k *K3sNode) GetCommandRole() string {
if k.role == common.RoleControlPlane ||
k.role == common.RoleControlPlaneHA ||
k.role == common.RoleControlPlaneClusterInit ||
k.role == common.RoleMaster ||
k.role == common.RoleMasterHA ||
k.role == common.RoleMasterInit {
return "server"
}
return "agent"
}

View File

@@ -6,103 +6,91 @@ import (
"github.com/kairos-io/kairos-sdk/machine"
"github.com/kairos-io/kairos-sdk/utils"
providerConfig "github.com/kairos-io/provider-kairos/v2/internal/provider/config"
common "github.com/kairos-io/provider-kairos/v2/internal/role"
service "github.com/mudler/edgevpn/api/client/service"
)
type ServiceDefinition interface {
Args() ([]string, error)
Env() map[string]string
EnvFile() string
K8sBin() string
Role() string
ServiceName() string
}
// Node represents any kubernetes node, regardless of its role
type Node interface {
// Core node functionality
GetIP() string
SetIP(string)
GetRole() string
SetRole(string)
GetDistro() string // k3s, k0s
K8sBin() string // Returns the path to the Kubernetes binary
type K8sControlPlane interface {
Args() ([]string, error)
ClusterInit() bool
DeployKubeVIP() error
Distro() string
EnvUnit() string
// Configuration
GetConfig() *providerConfig.Config
SetRoleConfig(*service.RoleConfig)
GetRoleConfig() *service.RoleConfig
// Service management
GetService() (machine.Service, error)
GetServiceName() string
GetEnvFile() string
GenerateEnv() map[string]string
GuessInterface()
HA() bool
IP() string
K8sBin() string
// Node operations
GenerateArgs() ([]string, error)
PropagateData() error
ProviderConfig() *providerConfig.Config
Role() string
RoleConfig() *service.RoleConfig
Service() (machine.Service, error)
SetIP(ip string)
SetRole(role string)
SetRoleConfig(c *service.RoleConfig)
GetToken() (string, error)
}
// ControlPlaneNode represents additional functionality specific to control plane nodes
type ControlPlaneNode interface {
Node
IsHA() bool
IsClusterInit() bool
SetupHAToken() error
Token() (string, error)
DeployKubeVIP() error
}
type K8sWorker interface {
Args() ([]string, error)
Distro() string
IP() string
K8sBin() string
ProviderConfig() *providerConfig.Config
Role() string
RoleConfig() *service.RoleConfig
Service() (machine.Service, error)
SetIP(ip string)
SetRole(role string)
SetRoleConfig(c *service.RoleConfig)
// WorkerNode represents additional functionality specific to worker nodes
type WorkerNode interface {
Node
SetupWorker(controlPlaneIP, nodeToken string) error
Token() (string, error)
}
func NewServiceDefinition(c *providerConfig.Config) (ServiceDefinition, error) {
func NewNode(config *providerConfig.Config, role string) (Node, error) {
switch {
case c.K3s.Enabled:
return &K3sControlPlane{providerConfig: c}, nil
case c.K0s.Enabled:
return &K0sControlPlane{providerConfig: c}, nil
case c.K3sAgent.Enabled:
return &K3sWorker{providerConfig: c}, nil
case c.K0sWorker.Enabled:
return &K0sWorker{providerConfig: c}, nil
// we don't know if it's a control plane or a worker
case config.K3s.Enabled:
base := &K3sNode{providerConfig: config}
if role == common.RoleWorker {
return &K3sWorker{K3sNode: base}, nil
}
return &K3sControlPlane{K3sNode: base}, nil
case config.K0s.Enabled:
base := &K0sNode{providerConfig: config}
if role == common.RoleWorker {
return &K0sWorker{K0sNode: base}, nil
}
return &K0sControlPlane{K0sNode: base}, nil
case utils.K3sBin() != "":
return &K3sControlPlane{providerConfig: c}, nil
base := &K3sNode{providerConfig: config}
if role == common.RoleWorker {
return &K3sWorker{K3sNode: base}, nil
}
return &K3sControlPlane{K3sNode: base}, nil
case utils.K0sBin() != "":
return &K0sControlPlane{providerConfig: c}, nil
base := &K0sNode{providerConfig: config}
if role == common.RoleWorker {
return &K0sWorker{K0sNode: base}, nil
}
return &K0sControlPlane{K0sNode: base}, nil
}
return nil, errors.New("no k8s distro found")
}
func NewK8sControlPlane(c *providerConfig.Config) (K8sControlPlane, error) {
switch {
case c.K3s.Enabled:
return &K3sControlPlane{providerConfig: c}, nil
case c.K0s.Enabled:
return &K0sControlPlane{providerConfig: c}, nil
case utils.K3sBin() != "":
return &K3sControlPlane{providerConfig: c}, nil
case utils.K0sBin() != "":
return &K0sControlPlane{providerConfig: c}, nil
}
return nil, errors.New("no k8s distro found")
// Helper function to convert Node to ControlPlaneNode
func AsControlPlane(n Node) (ControlPlaneNode, bool) {
cp, ok := n.(ControlPlaneNode)
return cp, ok
}
func NewK8sWorker(c *providerConfig.Config) (K8sWorker, error) {
switch {
case c.K3sAgent.Enabled:
return &K3sWorker{providerConfig: c}, nil
case c.K0sWorker.Enabled:
return &K0sWorker{providerConfig: c}, nil
case utils.K3sBin() != "":
return &K3sWorker{providerConfig: c}, nil
case utils.K0sBin() != "":
return &K0sWorker{providerConfig: c}, nil
}
return nil, errors.New("no k8s distro found")
// Helper function to convert Node to WorkerNode
func AsWorker(n Node) (WorkerNode, bool) {
w, ok := n.(WorkerNode)
return w, ok
}

View File

@@ -36,11 +36,16 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) role.Role { //nol
return nil
}
worker, err := NewK8sWorker(pconfig)
node, err := NewNode(pconfig, common.RoleWorker)
if err != nil {
return fmt.Errorf("failed to determine k8s distro: %w", err)
}
worker, ok := AsWorker(node)
if !ok {
return fmt.Errorf("failed to convert node to worker")
}
ip := guessIP(pconfig)
if ip != "" {
if err := c.Client.Set("ip", c.UUID, ip); err != nil {
@@ -52,7 +57,7 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) role.Role { //nol
worker.SetRoleConfig(c)
worker.SetIP(ip)
workerToken, _ := worker.Token()
workerToken, _ := worker.GetToken()
if workerToken == "" {
c.Logger.Info("worker token not there still..")
return nil
@@ -65,23 +70,26 @@ func Worker(cc *config.Config, pconfig *providerConfig.Config) role.Role { //nol
return err
}
k8sBin := worker.K8sBin()
k8sBin := utils.K3sBin()
if k8sBin == "" {
return fmt.Errorf("no %s binary found (?)", worker.Distro())
k8sBin = utils.K0sBin()
}
if k8sBin == "" {
return fmt.Errorf("no %s binary found (?)", worker.GetDistro())
}
args, err := worker.Args()
args, err := worker.GenerateArgs()
if err != nil {
return err
}
svc, err := worker.Service()
svc, err := worker.GetService()
if err != nil {
return err
}
c.Logger.Info(fmt.Sprintf("Configuring %s worker", worker.Distro()))
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", k8sBin, worker.Role(), strings.Join(args, " "))); err != nil {
c.Logger.Info(fmt.Sprintf("Configuring %s worker", worker.GetDistro()))
if err := svc.OverrideCmd(fmt.Sprintf("%s %s %s", k8sBin, worker.GetRole(), strings.Join(args, " "))); err != nil {
return err
}