From 6a9fad59f9f8aa31575060c7e4caf836967cde73 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Mon, 8 Jul 2024 07:15:09 -0600 Subject: [PATCH] fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions (#83) * fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions * ci: run unit tests --------- Signed-off-by: Tyler Gillson --- .github/workflows/pull_request.yaml | 18 +- main.go | 236 +------------------------ pkg/provider/provider.go | 261 ++++++++++++++++++++++++++++ pkg/provider/provider_test.go | 88 ++++++++++ 4 files changed, 355 insertions(+), 248 deletions(-) create mode 100644 pkg/provider/provider.go create mode 100644 pkg/provider/provider_test.go diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 6bf94ac..a41e10e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -19,19 +19,9 @@ jobs: with: version: "v0.6.30" - run: earthly --ci +lint - build-provider-package: + + test: runs-on: ubuntu-latest - permissions: - packages: write steps: - - uses: actions/checkout@v2 - - uses: docker-practice/actions-setup-docker@master - - uses: earthly/actions-setup@v1 - with: - version: "v0.6.30" - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - run: earthly --ci +provider-package-all-platforms --IMAGE_REPOSITORY=ghcr.io/kairos-io \ No newline at end of file + - uses: actions/checkout@v3 + - run: go test ./... \ No newline at end of file diff --git a/main.go b/main.go index f220226..4baff87 100644 --- a/main.go +++ b/main.go @@ -1,246 +1,14 @@ package main import ( - "encoding/json" - "fmt" - "net" "os" - "path/filepath" - "strings" "github.com/kairos-io/kairos-sdk/clusterplugin" - yip "github.com/mudler/yip/pkg/schema" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" - kyaml "sigs.k8s.io/yaml" - "github.com/kairos-io/provider-k3s/api" - "github.com/kairos-io/provider-k3s/pkg/constants" + "github.com/kairos-io/provider-k3s/pkg/provider" ) -const ( - configurationPath = "/etc/rancher/k3s/config.d" - containerdEnvConfigPath = "/etc/default" - localImagesPath = "/opt/content/images" - - serverSystemName = "k3s" - agentSystemName = "k3s-agent" - - bootBefore = "boot.before" - k8sNoProxy = ".svc,.svc.cluster,.svc.cluster.local" -) - -func clusterProvider(cluster clusterplugin.Cluster) yip.YipConfig { - k3sConfig := &api.K3sServerConfig{ - Token: cluster.ClusterToken, - } - userOptionConfig := cluster.Options - - logrus.Infof("current node role %s", cluster.Role) - logrus.Infof("received cluster options %s", cluster.Options) - - switch cluster.Role { - case clusterplugin.RoleInit: - k3sConfig.ClusterInit = true - k3sConfig.TLSSan = []string{cluster.ControlPlaneHost} - case clusterplugin.RoleControlPlane: - k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost) - k3sConfig.TLSSan = []string{cluster.ControlPlaneHost} - case clusterplugin.RoleWorker: - userOptionConfig = "" - k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost) - // Data received from upstream contains config for both control plane and worker. Thus, for worker, - // config is being filtered via unmarshal into agent config. - var agentCfg api.K3sAgentConfig - if err := yaml.Unmarshal([]byte(cluster.Options), &agentCfg); err == nil { - out, _ := yaml.Marshal(agentCfg) - userOptionConfig = string(out) - } else { - logrus.Fatalf("failed to un-marshal cluster options in k3s agent config %s", err) - } - } - - // if provided, parse additional K3s server options (which may override the above settings) - if cluster.ProviderOptions != nil { - providerOpts, err := yaml.Marshal(cluster.ProviderOptions) - if err != nil { - logrus.Fatalf("failed to marshal cluster.ProviderOptions: %v", err) - } - if err := yaml.Unmarshal(providerOpts, k3sConfig); err != nil { - logrus.Fatalf("failed to unmarshal cluster.ProviderOptions: %v", err) - } - logrus.Infof("applied cluster provider options: %+v", cluster.ProviderOptions) - } - - systemName := serverSystemName - if cluster.Role == clusterplugin.RoleWorker { - systemName = agentSystemName - } - - userOptions, _ := kyaml.YAMLToJSON([]byte(userOptionConfig)) - proxyOptions, _ := kyaml.YAMLToJSON([]byte(cluster.Options)) - options, _ := json.Marshal(k3sConfig) - - logrus.Infof("received cluster env %+v", cluster.Env) - - files := []yip.File{ - { - Path: filepath.Join(configurationPath, "90_userdata.yaml"), - Permissions: 0400, - Content: string(userOptions), - }, - { - Path: filepath.Join(configurationPath, "99_userdata.yaml"), - Permissions: 0400, - Content: string(options), - }, - } - - proxyValues := proxyEnv(proxyOptions, cluster.Env) - - if len(proxyValues) > 0 { - logrus.Infof("setting proxy values %s", proxyValues) - files = append(files, yip.File{ - Path: filepath.Join(containerdEnvConfigPath, systemName), - Permissions: 0400, - Content: proxyValues, - }) - } - - var stages []yip.Stage - - stages = append(stages, yip.Stage{ - Name: constants.InstallK3sConfigFiles, - Files: files, - Commands: []string{ - fmt.Sprintf("jq -s 'def flatten: reduce .[] as $i([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end); [.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value)' %s/*.yaml > /etc/rancher/k3s/config.yaml", configurationPath), - }, - }) - - var importStage yip.Stage - if cluster.ImportLocalImages { - if cluster.LocalImagesPath == "" { - cluster.LocalImagesPath = localImagesPath - } - - importStage = yip.Stage{ - Name: constants.ImportK3sImages, - Commands: []string{ - "chmod +x /opt/k3s/scripts/import.sh", - fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath), - }, - } - stages = append(stages, importStage) - } - - stages = append(stages, - yip.Stage{ - Name: constants.EnableOpenRCServices, - If: "[ -x /sbin/openrc-run ]", - Commands: []string{ - fmt.Sprintf("rc-update add %s default >/dev/null", systemName), - fmt.Sprintf("service %s start", systemName), - }, - }, - yip.Stage{ - Name: constants.EnableSystemdServices, - If: "[ -x /bin/systemctl ]", - Commands: []string{ - fmt.Sprintf("systemctl enable %s", systemName), - fmt.Sprintf("systemctl restart %s", systemName), - }, - }, - ) - - cfg := yip.YipConfig{ - Name: "K3s Kairos Cluster Provider", - Stages: map[string][]yip.Stage{ - bootBefore: stages, - }, - } - - return cfg -} - -func proxyEnv(proxyOptions []byte, proxyMap map[string]string) string { - var proxy []string - var noProxy string - var isProxyConfigured bool - - httpProxy := proxyMap["HTTP_PROXY"] - httpsProxy := proxyMap["HTTPS_PROXY"] - userNoProxy := proxyMap["NO_PROXY"] - - defaultNoProxy := getDefaultNoProxy(proxyOptions) - logrus.Infof("setting default no proxy to %s", defaultNoProxy) - - if len(httpProxy) > 0 { - proxy = append(proxy, fmt.Sprintf("HTTP_PROXY=%s", httpProxy)) - proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTP_PROXY=%s", httpProxy)) - isProxyConfigured = true - } - - if len(httpsProxy) > 0 { - proxy = append(proxy, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy)) - proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTPS_PROXY=%s", httpsProxy)) - isProxyConfigured = true - } - - if isProxyConfigured { - noProxy = defaultNoProxy - } - - if len(userNoProxy) > 0 { - noProxy = noProxy + "," + userNoProxy - } - - if len(noProxy) > 0 { - proxy = append(proxy, fmt.Sprintf("NO_PROXY=%s", noProxy)) - proxy = append(proxy, fmt.Sprintf("CONTAINERD_NO_PROXY=%s", noProxy)) - } - - return strings.Join(proxy, "\n") -} - -func getDefaultNoProxy(proxyOptions []byte) string { - var noProxy string - - data := make(map[string]interface{}) - err := json.Unmarshal(proxyOptions, &data) - if err != nil { - logrus.Fatalf("error while unmarshalling user options %s", err) - } - - if data != nil { - clusterCIDR := data["cluster-cidr"].(string) - serviceCIDR := data["service-cidr"].(string) - - if len(clusterCIDR) > 0 { - noProxy = noProxy + "," + clusterCIDR - } - if len(serviceCIDR) > 0 { - noProxy = noProxy + "," + serviceCIDR - } - } - noProxy = noProxy + "," + getNodeCIDR() + "," + k8sNoProxy - - return noProxy -} - -func getNodeCIDR() string { - addrs, _ := net.InterfaceAddrs() - var result string - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - result = addr.String() - break - } - } - } - return result -} - func main() { f, err := os.OpenFile("/var/log/provider-k3s.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { @@ -249,7 +17,7 @@ func main() { logrus.SetOutput(f) plugin := clusterplugin.ClusterPlugin{ - Provider: clusterProvider, + Provider: provider.ClusterProvider, } if err := plugin.Run(); err != nil { diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go new file mode 100644 index 0000000..aa44faf --- /dev/null +++ b/pkg/provider/provider.go @@ -0,0 +1,261 @@ +package provider + +import ( + "encoding/json" + "fmt" + "net" + "path/filepath" + "strings" + + "github.com/kairos-io/kairos-sdk/clusterplugin" + yip "github.com/mudler/yip/pkg/schema" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + kyaml "sigs.k8s.io/yaml" + + "github.com/kairos-io/provider-k3s/api" + "github.com/kairos-io/provider-k3s/pkg/constants" +) + +const ( + configurationPath = "/etc/rancher/k3s/config.d" + containerdEnvConfigPath = "/etc/default" + localImagesPath = "/opt/content/images" + + serverSystemName = "k3s" + agentSystemName = "k3s-agent" + + bootBefore = "boot.before" + k8sNoProxy = ".svc,.svc.cluster,.svc.cluster.local" +) + +func ClusterProvider(cluster clusterplugin.Cluster) yip.YipConfig { + logrus.Infof("current node role %s", cluster.Role) + logrus.Infof("received cluster env %+v", cluster.Env) + logrus.Infof("received cluster options %s", cluster.Options) + + systemName := serverSystemName + if cluster.Role == clusterplugin.RoleWorker { + systemName = agentSystemName + } + + cfg := yip.YipConfig{ + Name: "K3s Kairos Cluster Provider", + Stages: map[string][]yip.Stage{ + bootBefore: parseStages(cluster, parseFiles(cluster, systemName), systemName), + }, + } + + return cfg +} + +func parseOptions(cluster clusterplugin.Cluster) ([]byte, []byte, []byte) { + k3sConfig := &api.K3sServerConfig{ + Token: cluster.ClusterToken, + } + userOptionConfig := cluster.Options + + switch cluster.Role { + case clusterplugin.RoleInit: + k3sConfig.ClusterInit = true + k3sConfig.TLSSan = []string{cluster.ControlPlaneHost} + case clusterplugin.RoleControlPlane: + k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost) + k3sConfig.TLSSan = []string{cluster.ControlPlaneHost} + case clusterplugin.RoleWorker: + userOptionConfig = "" + k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost) + // Data received from upstream contains config for both control plane and worker. Thus, for worker, + // config is being filtered via unmarshal into agent config. + var agentCfg api.K3sAgentConfig + if err := yaml.Unmarshal([]byte(cluster.Options), &agentCfg); err == nil { + out, _ := yaml.Marshal(agentCfg) + userOptionConfig = string(out) + } else { + logrus.Fatalf("failed to un-marshal cluster options in k3s agent config %s", err) + } + } + + userOptions, _ := kyaml.YAMLToJSON([]byte(userOptionConfig)) + proxyOptions, _ := kyaml.YAMLToJSON([]byte(cluster.Options)) + options, _ := json.Marshal(k3sConfig) + + // if provided, parse additional K3s server options (which may override the above settings) + if cluster.ProviderOptions != nil && len(cluster.ProviderOptions) > 0 { + logrus.Infof("applying cluster provider options: %+v", cluster.ProviderOptions) + + providerOpts, err := yaml.Marshal(cluster.ProviderOptions) + if err != nil { + logrus.Fatalf("failed to marshal cluster.ProviderOptions: %v", err) + } + if err := yaml.Unmarshal(providerOpts, k3sConfig); err != nil { + logrus.Fatalf("failed to unmarshal cluster.ProviderOptions: %v", err) + } + options, _ = json.Marshal(k3sConfig) + + if v, ok := cluster.ProviderOptions[constants.ClusterInit]; ok && v == "no" { + // Manually set cluster-init to false, as it's dropped by the above marshal. + // We want to omit this field in all other scenarios, hence this special, ugly case. + override := []byte(`{"cluster-init":false,`) + options = append(override, options[1:]...) + } + } + + return options, proxyOptions, userOptions +} + +func parseFiles(cluster clusterplugin.Cluster, systemName string) []yip.File { + options, proxyOptions, userOptions := parseOptions(cluster) + + files := []yip.File{ + { + Path: filepath.Join(configurationPath, "90_userdata.yaml"), + Permissions: 0400, + Content: string(userOptions), + }, + { + Path: filepath.Join(configurationPath, "99_userdata.yaml"), + Permissions: 0400, + Content: string(options), + }, + } + + proxyValues := proxyEnv(proxyOptions, cluster.Env) + + if len(proxyValues) > 0 { + logrus.Infof("setting proxy values %s", proxyValues) + files = append(files, yip.File{ + Path: filepath.Join(containerdEnvConfigPath, systemName), + Permissions: 0400, + Content: proxyValues, + }) + } + + return files +} + +func parseStages(cluster clusterplugin.Cluster, files []yip.File, systemName string) []yip.Stage { + var stages []yip.Stage + + stages = append(stages, yip.Stage{ + Name: constants.InstallK3sConfigFiles, + Files: files, + Commands: []string{ + fmt.Sprintf("jq -s 'def flatten: reduce .[] as $i([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end); [.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value)' %s/*.yaml > /etc/rancher/k3s/config.yaml", configurationPath), + }, + }) + + if cluster.ImportLocalImages { + if cluster.LocalImagesPath == "" { + cluster.LocalImagesPath = localImagesPath + } + importStage := yip.Stage{ + Name: constants.ImportK3sImages, + Commands: []string{ + "chmod +x /opt/k3s/scripts/import.sh", + fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath), + }, + } + stages = append(stages, importStage) + } + + stages = append(stages, + yip.Stage{ + Name: constants.EnableOpenRCServices, + If: "[ -x /sbin/openrc-run ]", + Commands: []string{ + fmt.Sprintf("rc-update add %s default >/dev/null", systemName), + fmt.Sprintf("service %s start", systemName), + }, + }, + yip.Stage{ + Name: constants.EnableSystemdServices, + If: "[ -x /bin/systemctl ]", + Commands: []string{ + fmt.Sprintf("systemctl enable %s", systemName), + fmt.Sprintf("systemctl restart %s", systemName), + }, + }, + ) + + return stages +} + +func proxyEnv(proxyOptions []byte, proxyMap map[string]string) string { + var proxy []string + var noProxy string + var isProxyConfigured bool + + httpProxy := proxyMap["HTTP_PROXY"] + httpsProxy := proxyMap["HTTPS_PROXY"] + userNoProxy := proxyMap["NO_PROXY"] + + defaultNoProxy := getDefaultNoProxy(proxyOptions) + logrus.Infof("setting default no proxy to %s", defaultNoProxy) + + if len(httpProxy) > 0 { + proxy = append(proxy, fmt.Sprintf("HTTP_PROXY=%s", httpProxy)) + proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTP_PROXY=%s", httpProxy)) + isProxyConfigured = true + } + + if len(httpsProxy) > 0 { + proxy = append(proxy, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy)) + proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTPS_PROXY=%s", httpsProxy)) + isProxyConfigured = true + } + + if isProxyConfigured { + noProxy = defaultNoProxy + } + + if len(userNoProxy) > 0 { + noProxy = noProxy + "," + userNoProxy + } + + if len(noProxy) > 0 { + proxy = append(proxy, fmt.Sprintf("NO_PROXY=%s", noProxy)) + proxy = append(proxy, fmt.Sprintf("CONTAINERD_NO_PROXY=%s", noProxy)) + } + + return strings.Join(proxy, "\n") +} + +func getDefaultNoProxy(proxyOptions []byte) string { + var noProxy string + + data := make(map[string]interface{}) + err := json.Unmarshal(proxyOptions, &data) + if err != nil { + logrus.Fatalf("error while unmarshalling user options %s", err) + } + + if data != nil { + clusterCIDR := data["cluster-cidr"].(string) + serviceCIDR := data["service-cidr"].(string) + + if len(clusterCIDR) > 0 { + noProxy = noProxy + "," + clusterCIDR + } + if len(serviceCIDR) > 0 { + noProxy = noProxy + "," + serviceCIDR + } + } + noProxy = noProxy + "," + getNodeCIDR() + "," + k8sNoProxy + + return noProxy +} + +func getNodeCIDR() string { + addrs, _ := net.InterfaceAddrs() + var result string + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + result = addr.String() + break + } + } + } + return result +} diff --git a/pkg/provider/provider_test.go b/pkg/provider/provider_test.go new file mode 100644 index 0000000..c847184 --- /dev/null +++ b/pkg/provider/provider_test.go @@ -0,0 +1,88 @@ +package provider + +import ( + "bytes" + "testing" + + "github.com/kairos-io/kairos-sdk/clusterplugin" +) + +func Test_parseOptions(t *testing.T) { + tests := []struct { + name string + cluster clusterplugin.Cluster + expectedOptions []byte + expectedProxyOptions []byte + expectedUserOptions []byte + }{ + { + name: "Empty input", + cluster: clusterplugin.Cluster{}, + expectedOptions: []byte(`{}`), + expectedProxyOptions: []byte(`null`), + expectedUserOptions: []byte(`null`), + }, + { + name: "Init: Standard", + cluster: clusterplugin.Cluster{ + ClusterToken: "token", + ControlPlaneHost: "localhost", + Role: "init", + }, + expectedOptions: []byte(`{"cluster-init":true,"token":"token","tls-san":["localhost"]}`), + expectedProxyOptions: []byte(`null`), + expectedUserOptions: []byte(`null`), + }, + { + name: "Init: 2-Node", + cluster: clusterplugin.Cluster{ + ClusterToken: "token", + ControlPlaneHost: "localhost", + Role: "init", + ProviderOptions: map[string]string{ + "cluster-init": "no", + "datastore-endpoint": "localhost:2379", + }, + }, + expectedOptions: []byte(`{"cluster-init":false,"token":"token","tls-san":["localhost"],"datastore-endpoint":"localhost:2379"}`), + expectedProxyOptions: []byte(`null`), + expectedUserOptions: []byte(`null`), + }, + { + name: "Control Plane", + cluster: clusterplugin.Cluster{ + ClusterToken: "token", + ControlPlaneHost: "localhost", + Role: "controlplane", + }, + expectedOptions: []byte(`{"token":"token","server":"https://localhost:6443","tls-san":["localhost"]}`), + expectedProxyOptions: []byte(`null`), + expectedUserOptions: []byte(`null`), + }, + { + name: "Worker", + cluster: clusterplugin.Cluster{ + ClusterToken: "token", + ControlPlaneHost: "localhost", + Role: "worker", + }, + expectedOptions: []byte(`{"token":"token","server":"https://localhost:6443"}`), + expectedProxyOptions: []byte(`null`), + expectedUserOptions: []byte(`{}`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options, proxyOptions, userOptions := parseOptions(tt.cluster) + if !bytes.Equal(options, tt.expectedOptions) { + t.Errorf("parseOptions() options = %v, want %v", string(options), string(tt.expectedOptions)) + } + if !bytes.Equal(proxyOptions, tt.expectedProxyOptions) { + t.Errorf("parseOptions() proxyOptions = %v, want %v", string(proxyOptions), string(tt.expectedProxyOptions)) + } + if !bytes.Equal(userOptions, tt.expectedUserOptions) { + t.Errorf("parseOptions() userOptions = %v, want %v", string(userOptions), string(tt.expectedUserOptions)) + } + }) + } +}