fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions (#83)

* fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions
* ci: run unit tests
---------

Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com>

.github/workflows/pull_request.yaml

@@ -19,19 +19,9 @@ jobs:
         with:
           version: "v0.6.30"
       - run: earthly --ci +lint
-  build-provider-package:
+
+  test:
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: docker-practice/actions-setup-docker@master
+      - run: go test ./...
-      - uses: earthly/actions-setup@v1
-        with:
-          version: "v0.6.30"
-      - uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }} 
-      - run: earthly --ci  +provider-package-all-platforms --IMAGE_REPOSITORY=ghcr.io/kairos-io

main.go

@@ -1,246 +1,14 @@
 package main
 
 import (
-	"encoding/json"
-	"fmt"
-	"net"
 	"os"
-	"path/filepath"
-	"strings"
 
 	"github.com/kairos-io/kairos-sdk/clusterplugin"
-	yip "github.com/mudler/yip/pkg/schema"
 	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
-	kyaml "sigs.k8s.io/yaml"
 
-	"github.com/kairos-io/provider-k3s/api"
+	"github.com/kairos-io/provider-k3s/pkg/provider"
-	"github.com/kairos-io/provider-k3s/pkg/constants"
 )
 
-const (
-	configurationPath       = "/etc/rancher/k3s/config.d"
-	containerdEnvConfigPath = "/etc/default"
-	localImagesPath         = "/opt/content/images"
-
-	serverSystemName = "k3s"
-	agentSystemName  = "k3s-agent"
-
-	bootBefore = "boot.before"
-	k8sNoProxy = ".svc,.svc.cluster,.svc.cluster.local"
-)
-
-func clusterProvider(cluster clusterplugin.Cluster) yip.YipConfig {
-	k3sConfig := &api.K3sServerConfig{
-		Token: cluster.ClusterToken,
-	}
-	userOptionConfig := cluster.Options
-
-	logrus.Infof("current node role %s", cluster.Role)
-	logrus.Infof("received cluster options %s", cluster.Options)
-
-	switch cluster.Role {
-	case clusterplugin.RoleInit:
-		k3sConfig.ClusterInit = true
-		k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
-	case clusterplugin.RoleControlPlane:
-		k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
-		k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
-	case clusterplugin.RoleWorker:
-		userOptionConfig = ""
-		k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
-		// Data received from upstream contains config for both control plane and worker. Thus, for worker,
-		// config is being filtered via unmarshal into agent config.
-		var agentCfg api.K3sAgentConfig
-		if err := yaml.Unmarshal([]byte(cluster.Options), &agentCfg); err == nil {
-			out, _ := yaml.Marshal(agentCfg)
-			userOptionConfig = string(out)
-		} else {
-			logrus.Fatalf("failed to un-marshal cluster options in k3s agent config %s", err)
-		}
-	}
-
-	// if provided, parse additional K3s server options (which may override the above settings)
-	if cluster.ProviderOptions != nil {
-		providerOpts, err := yaml.Marshal(cluster.ProviderOptions)
-		if err != nil {
-			logrus.Fatalf("failed to marshal cluster.ProviderOptions: %v", err)
-		}
-		if err := yaml.Unmarshal(providerOpts, k3sConfig); err != nil {
-			logrus.Fatalf("failed to unmarshal cluster.ProviderOptions: %v", err)
-		}
-		logrus.Infof("applied cluster provider options: %+v", cluster.ProviderOptions)
-	}
-
-	systemName := serverSystemName
-	if cluster.Role == clusterplugin.RoleWorker {
-		systemName = agentSystemName
-	}
-
-	userOptions, _ := kyaml.YAMLToJSON([]byte(userOptionConfig))
-	proxyOptions, _ := kyaml.YAMLToJSON([]byte(cluster.Options))
-	options, _ := json.Marshal(k3sConfig)
-
-	logrus.Infof("received cluster env %+v", cluster.Env)
-
-	files := []yip.File{
-		{
-			Path:        filepath.Join(configurationPath, "90_userdata.yaml"),
-			Permissions: 0400,
-			Content:     string(userOptions),
-		},
-		{
-			Path:        filepath.Join(configurationPath, "99_userdata.yaml"),
-			Permissions: 0400,
-			Content:     string(options),
-		},
-	}
-
-	proxyValues := proxyEnv(proxyOptions, cluster.Env)
-
-	if len(proxyValues) > 0 {
-		logrus.Infof("setting proxy values %s", proxyValues)
-		files = append(files, yip.File{
-			Path:        filepath.Join(containerdEnvConfigPath, systemName),
-			Permissions: 0400,
-			Content:     proxyValues,
-		})
-	}
-
-	var stages []yip.Stage
-
-	stages = append(stages, yip.Stage{
-		Name:  constants.InstallK3sConfigFiles,
-		Files: files,
-		Commands: []string{
-			fmt.Sprintf("jq -s 'def flatten: reduce .[] as $i([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end); [.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value)' %s/*.yaml > /etc/rancher/k3s/config.yaml", configurationPath),
-		},
-	})
-
-	var importStage yip.Stage
-	if cluster.ImportLocalImages {
-		if cluster.LocalImagesPath == "" {
-			cluster.LocalImagesPath = localImagesPath
-		}
-
-		importStage = yip.Stage{
-			Name: constants.ImportK3sImages,
-			Commands: []string{
-				"chmod +x /opt/k3s/scripts/import.sh",
-				fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath),
-			},
-		}
-		stages = append(stages, importStage)
-	}
-
-	stages = append(stages,
-		yip.Stage{
-			Name: constants.EnableOpenRCServices,
-			If:   "[ -x /sbin/openrc-run ]",
-			Commands: []string{
-				fmt.Sprintf("rc-update add %s default >/dev/null", systemName),
-				fmt.Sprintf("service %s start", systemName),
-			},
-		},
-		yip.Stage{
-			Name: constants.EnableSystemdServices,
-			If:   "[ -x /bin/systemctl ]",
-			Commands: []string{
-				fmt.Sprintf("systemctl enable %s", systemName),
-				fmt.Sprintf("systemctl restart %s", systemName),
-			},
-		},
-	)
-
-	cfg := yip.YipConfig{
-		Name: "K3s Kairos Cluster Provider",
-		Stages: map[string][]yip.Stage{
-			bootBefore: stages,
-		},
-	}
-
-	return cfg
-}
-
-func proxyEnv(proxyOptions []byte, proxyMap map[string]string) string {
-	var proxy []string
-	var noProxy string
-	var isProxyConfigured bool
-
-	httpProxy := proxyMap["HTTP_PROXY"]
-	httpsProxy := proxyMap["HTTPS_PROXY"]
-	userNoProxy := proxyMap["NO_PROXY"]
-
-	defaultNoProxy := getDefaultNoProxy(proxyOptions)
-	logrus.Infof("setting default no proxy to %s", defaultNoProxy)
-
-	if len(httpProxy) > 0 {
-		proxy = append(proxy, fmt.Sprintf("HTTP_PROXY=%s", httpProxy))
-		proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTP_PROXY=%s", httpProxy))
-		isProxyConfigured = true
-	}
-
-	if len(httpsProxy) > 0 {
-		proxy = append(proxy, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy))
-		proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTPS_PROXY=%s", httpsProxy))
-		isProxyConfigured = true
-	}
-
-	if isProxyConfigured {
-		noProxy = defaultNoProxy
-	}
-
-	if len(userNoProxy) > 0 {
-		noProxy = noProxy + "," + userNoProxy
-	}
-
-	if len(noProxy) > 0 {
-		proxy = append(proxy, fmt.Sprintf("NO_PROXY=%s", noProxy))
-		proxy = append(proxy, fmt.Sprintf("CONTAINERD_NO_PROXY=%s", noProxy))
-	}
-
-	return strings.Join(proxy, "\n")
-}
-
-func getDefaultNoProxy(proxyOptions []byte) string {
-	var noProxy string
-
-	data := make(map[string]interface{})
-	err := json.Unmarshal(proxyOptions, &data)
-	if err != nil {
-		logrus.Fatalf("error while unmarshalling user options %s", err)
-	}
-
-	if data != nil {
-		clusterCIDR := data["cluster-cidr"].(string)
-		serviceCIDR := data["service-cidr"].(string)
-
-		if len(clusterCIDR) > 0 {
-			noProxy = noProxy + "," + clusterCIDR
-		}
-		if len(serviceCIDR) > 0 {
-			noProxy = noProxy + "," + serviceCIDR
-		}
-	}
-	noProxy = noProxy + "," + getNodeCIDR() + "," + k8sNoProxy
-
-	return noProxy
-}
-
-func getNodeCIDR() string {
-	addrs, _ := net.InterfaceAddrs()
-	var result string
-	for _, addr := range addrs {
-		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				result = addr.String()
-				break
-			}
-		}
-	}
-	return result
-}
-
 func main() {
 	f, err := os.OpenFile("/var/log/provider-k3s.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
 	if err != nil {
@@ -249,7 +17,7 @@ func main() {
 	logrus.SetOutput(f)
 
 	plugin := clusterplugin.ClusterPlugin{
-		Provider: clusterProvider,
+		Provider: provider.ClusterProvider,
 	}
 
 	if err := plugin.Run(); err != nil {

pkg/provider/provider.go

@@ -0,0 +1,261 @@
+package provider
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"path/filepath"
+	"strings"
+
+	"github.com/kairos-io/kairos-sdk/clusterplugin"
+	yip "github.com/mudler/yip/pkg/schema"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+	kyaml "sigs.k8s.io/yaml"
+
+	"github.com/kairos-io/provider-k3s/api"
+	"github.com/kairos-io/provider-k3s/pkg/constants"
+)
+
+const (
+	configurationPath       = "/etc/rancher/k3s/config.d"
+	containerdEnvConfigPath = "/etc/default"
+	localImagesPath         = "/opt/content/images"
+
+	serverSystemName = "k3s"
+	agentSystemName  = "k3s-agent"
+
+	bootBefore = "boot.before"
+	k8sNoProxy = ".svc,.svc.cluster,.svc.cluster.local"
+)
+
+func ClusterProvider(cluster clusterplugin.Cluster) yip.YipConfig {
+	logrus.Infof("current node role %s", cluster.Role)
+	logrus.Infof("received cluster env %+v", cluster.Env)
+	logrus.Infof("received cluster options %s", cluster.Options)
+
+	systemName := serverSystemName
+	if cluster.Role == clusterplugin.RoleWorker {
+		systemName = agentSystemName
+	}
+
+	cfg := yip.YipConfig{
+		Name: "K3s Kairos Cluster Provider",
+		Stages: map[string][]yip.Stage{
+			bootBefore: parseStages(cluster, parseFiles(cluster, systemName), systemName),
+		},
+	}
+
+	return cfg
+}
+
+func parseOptions(cluster clusterplugin.Cluster) ([]byte, []byte, []byte) {
+	k3sConfig := &api.K3sServerConfig{
+		Token: cluster.ClusterToken,
+	}
+	userOptionConfig := cluster.Options
+
+	switch cluster.Role {
+	case clusterplugin.RoleInit:
+		k3sConfig.ClusterInit = true
+		k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
+	case clusterplugin.RoleControlPlane:
+		k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
+		k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
+	case clusterplugin.RoleWorker:
+		userOptionConfig = ""
+		k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
+		// Data received from upstream contains config for both control plane and worker. Thus, for worker,
+		// config is being filtered via unmarshal into agent config.
+		var agentCfg api.K3sAgentConfig
+		if err := yaml.Unmarshal([]byte(cluster.Options), &agentCfg); err == nil {
+			out, _ := yaml.Marshal(agentCfg)
+			userOptionConfig = string(out)
+		} else {
+			logrus.Fatalf("failed to un-marshal cluster options in k3s agent config %s", err)
+		}
+	}
+
+	userOptions, _ := kyaml.YAMLToJSON([]byte(userOptionConfig))
+	proxyOptions, _ := kyaml.YAMLToJSON([]byte(cluster.Options))
+	options, _ := json.Marshal(k3sConfig)
+
+	// if provided, parse additional K3s server options (which may override the above settings)
+	if cluster.ProviderOptions != nil && len(cluster.ProviderOptions) > 0 {
+		logrus.Infof("applying cluster provider options: %+v", cluster.ProviderOptions)
+
+		providerOpts, err := yaml.Marshal(cluster.ProviderOptions)
+		if err != nil {
+			logrus.Fatalf("failed to marshal cluster.ProviderOptions: %v", err)
+		}
+		if err := yaml.Unmarshal(providerOpts, k3sConfig); err != nil {
+			logrus.Fatalf("failed to unmarshal cluster.ProviderOptions: %v", err)
+		}
+		options, _ = json.Marshal(k3sConfig)
+
+		if v, ok := cluster.ProviderOptions[constants.ClusterInit]; ok && v == "no" {
+			// Manually set cluster-init to false, as it's dropped by the above marshal.
+			// We want to omit this field in all other scenarios, hence this special, ugly case.
+			override := []byte(`{"cluster-init":false,`)
+			options = append(override, options[1:]...)
+		}
+	}
+
+	return options, proxyOptions, userOptions
+}
+
+func parseFiles(cluster clusterplugin.Cluster, systemName string) []yip.File {
+	options, proxyOptions, userOptions := parseOptions(cluster)
+
+	files := []yip.File{
+		{
+			Path:        filepath.Join(configurationPath, "90_userdata.yaml"),
+			Permissions: 0400,
+			Content:     string(userOptions),
+		},
+		{
+			Path:        filepath.Join(configurationPath, "99_userdata.yaml"),
+			Permissions: 0400,
+			Content:     string(options),
+		},
+	}
+
+	proxyValues := proxyEnv(proxyOptions, cluster.Env)
+
+	if len(proxyValues) > 0 {
+		logrus.Infof("setting proxy values %s", proxyValues)
+		files = append(files, yip.File{
+			Path:        filepath.Join(containerdEnvConfigPath, systemName),
+			Permissions: 0400,
+			Content:     proxyValues,
+		})
+	}
+
+	return files
+}
+
+func parseStages(cluster clusterplugin.Cluster, files []yip.File, systemName string) []yip.Stage {
+	var stages []yip.Stage
+
+	stages = append(stages, yip.Stage{
+		Name:  constants.InstallK3sConfigFiles,
+		Files: files,
+		Commands: []string{
+			fmt.Sprintf("jq -s 'def flatten: reduce .[] as $i([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end); [.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value)' %s/*.yaml > /etc/rancher/k3s/config.yaml", configurationPath),
+		},
+	})
+
+	if cluster.ImportLocalImages {
+		if cluster.LocalImagesPath == "" {
+			cluster.LocalImagesPath = localImagesPath
+		}
+		importStage := yip.Stage{
+			Name: constants.ImportK3sImages,
+			Commands: []string{
+				"chmod +x /opt/k3s/scripts/import.sh",
+				fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath),
+			},
+		}
+		stages = append(stages, importStage)
+	}
+
+	stages = append(stages,
+		yip.Stage{
+			Name: constants.EnableOpenRCServices,
+			If:   "[ -x /sbin/openrc-run ]",
+			Commands: []string{
+				fmt.Sprintf("rc-update add %s default >/dev/null", systemName),
+				fmt.Sprintf("service %s start", systemName),
+			},
+		},
+		yip.Stage{
+			Name: constants.EnableSystemdServices,
+			If:   "[ -x /bin/systemctl ]",
+			Commands: []string{
+				fmt.Sprintf("systemctl enable %s", systemName),
+				fmt.Sprintf("systemctl restart %s", systemName),
+			},
+		},
+	)
+
+	return stages
+}
+
+func proxyEnv(proxyOptions []byte, proxyMap map[string]string) string {
+	var proxy []string
+	var noProxy string
+	var isProxyConfigured bool
+
+	httpProxy := proxyMap["HTTP_PROXY"]
+	httpsProxy := proxyMap["HTTPS_PROXY"]
+	userNoProxy := proxyMap["NO_PROXY"]
+
+	defaultNoProxy := getDefaultNoProxy(proxyOptions)
+	logrus.Infof("setting default no proxy to %s", defaultNoProxy)
+
+	if len(httpProxy) > 0 {
+		proxy = append(proxy, fmt.Sprintf("HTTP_PROXY=%s", httpProxy))
+		proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTP_PROXY=%s", httpProxy))
+		isProxyConfigured = true
+	}
+
+	if len(httpsProxy) > 0 {
+		proxy = append(proxy, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy))
+		proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTPS_PROXY=%s", httpsProxy))
+		isProxyConfigured = true
+	}
+
+	if isProxyConfigured {
+		noProxy = defaultNoProxy
+	}
+
+	if len(userNoProxy) > 0 {
+		noProxy = noProxy + "," + userNoProxy
+	}
+
+	if len(noProxy) > 0 {
+		proxy = append(proxy, fmt.Sprintf("NO_PROXY=%s", noProxy))
+		proxy = append(proxy, fmt.Sprintf("CONTAINERD_NO_PROXY=%s", noProxy))
+	}
+
+	return strings.Join(proxy, "\n")
+}
+
+func getDefaultNoProxy(proxyOptions []byte) string {
+	var noProxy string
+
+	data := make(map[string]interface{})
+	err := json.Unmarshal(proxyOptions, &data)
+	if err != nil {
+		logrus.Fatalf("error while unmarshalling user options %s", err)
+	}
+
+	if data != nil {
+		clusterCIDR := data["cluster-cidr"].(string)
+		serviceCIDR := data["service-cidr"].(string)
+
+		if len(clusterCIDR) > 0 {
+			noProxy = noProxy + "," + clusterCIDR
+		}
+		if len(serviceCIDR) > 0 {
+			noProxy = noProxy + "," + serviceCIDR
+		}
+	}
+	noProxy = noProxy + "," + getNodeCIDR() + "," + k8sNoProxy
+
+	return noProxy
+}
+
+func getNodeCIDR() string {
+	addrs, _ := net.InterfaceAddrs()
+	var result string
+	for _, addr := range addrs {
+		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+			if ipnet.IP.To4() != nil {
+				result = addr.String()
+				break
+			}
+		}
+	}
+	return result
+}

pkg/provider/provider_test.go

@@ -0,0 +1,88 @@
+package provider
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/kairos-io/kairos-sdk/clusterplugin"
+)
+
+func Test_parseOptions(t *testing.T) {
+	tests := []struct {
+		name                 string
+		cluster              clusterplugin.Cluster
+		expectedOptions      []byte
+		expectedProxyOptions []byte
+		expectedUserOptions  []byte
+	}{
+		{
+			name:                 "Empty input",
+			cluster:              clusterplugin.Cluster{},
+			expectedOptions:      []byte(`{}`),
+			expectedProxyOptions: []byte(`null`),
+			expectedUserOptions:  []byte(`null`),
+		},
+		{
+			name: "Init: Standard",
+			cluster: clusterplugin.Cluster{
+				ClusterToken:     "token",
+				ControlPlaneHost: "localhost",
+				Role:             "init",
+			},
+			expectedOptions:      []byte(`{"cluster-init":true,"token":"token","tls-san":["localhost"]}`),
+			expectedProxyOptions: []byte(`null`),
+			expectedUserOptions:  []byte(`null`),
+		},
+		{
+			name: "Init: 2-Node",
+			cluster: clusterplugin.Cluster{
+				ClusterToken:     "token",
+				ControlPlaneHost: "localhost",
+				Role:             "init",
+				ProviderOptions: map[string]string{
+					"cluster-init":       "no",
+					"datastore-endpoint": "localhost:2379",
+				},
+			},
+			expectedOptions:      []byte(`{"cluster-init":false,"token":"token","tls-san":["localhost"],"datastore-endpoint":"localhost:2379"}`),
+			expectedProxyOptions: []byte(`null`),
+			expectedUserOptions:  []byte(`null`),
+		},
+		{
+			name: "Control Plane",
+			cluster: clusterplugin.Cluster{
+				ClusterToken:     "token",
+				ControlPlaneHost: "localhost",
+				Role:             "controlplane",
+			},
+			expectedOptions:      []byte(`{"token":"token","server":"https://localhost:6443","tls-san":["localhost"]}`),
+			expectedProxyOptions: []byte(`null`),
+			expectedUserOptions:  []byte(`null`),
+		},
+		{
+			name: "Worker",
+			cluster: clusterplugin.Cluster{
+				ClusterToken:     "token",
+				ControlPlaneHost: "localhost",
+				Role:             "worker",
+			},
+			expectedOptions:      []byte(`{"token":"token","server":"https://localhost:6443"}`),
+			expectedProxyOptions: []byte(`null`),
+			expectedUserOptions:  []byte(`{}`),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			options, proxyOptions, userOptions := parseOptions(tt.cluster)
+			if !bytes.Equal(options, tt.expectedOptions) {
+				t.Errorf("parseOptions() options = %v, want %v", string(options), string(tt.expectedOptions))
+			}
+			if !bytes.Equal(proxyOptions, tt.expectedProxyOptions) {
+				t.Errorf("parseOptions() proxyOptions = %v, want %v", string(proxyOptions), string(tt.expectedProxyOptions))
+			}
+			if !bytes.Equal(userOptions, tt.expectedUserOptions) {
+				t.Errorf("parseOptions() userOptions = %v, want %v", string(userOptions), string(tt.expectedUserOptions))
+			}
+		})
+	}
+}