mirror of
https://github.com/kairos-io/provider-k3s.git
synced 2025-10-20 04:58:39 +00:00
fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions (#83)
* fix: set 'cluster-init: false' when explicitly disabled via ProviderOptions * ci: run unit tests --------- Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com>
This commit is contained in:
Tyler Gillson
GitHub
261
pkg/provider/provider.go
Normal file
261
pkg/provider/provider.go
Normal file
@@ -0,0 +1,261 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/kairos-io/kairos-sdk/clusterplugin"
|
||||
yip "github.com/mudler/yip/pkg/schema"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v3"
|
||||
kyaml "sigs.k8s.io/yaml"
|
||||
|
||||
"github.com/kairos-io/provider-k3s/api"
|
||||
"github.com/kairos-io/provider-k3s/pkg/constants"
|
||||
)
|
||||
|
||||
const (
|
||||
configurationPath = "/etc/rancher/k3s/config.d"
|
||||
containerdEnvConfigPath = "/etc/default"
|
||||
localImagesPath = "/opt/content/images"
|
||||
|
||||
serverSystemName = "k3s"
|
||||
agentSystemName = "k3s-agent"
|
||||
|
||||
bootBefore = "boot.before"
|
||||
k8sNoProxy = ".svc,.svc.cluster,.svc.cluster.local"
|
||||
)
|
||||
|
||||
func ClusterProvider(cluster clusterplugin.Cluster) yip.YipConfig {
|
||||
logrus.Infof("current node role %s", cluster.Role)
|
||||
logrus.Infof("received cluster env %+v", cluster.Env)
|
||||
logrus.Infof("received cluster options %s", cluster.Options)
|
||||
|
||||
systemName := serverSystemName
|
||||
if cluster.Role == clusterplugin.RoleWorker {
|
||||
systemName = agentSystemName
|
||||
}
|
||||
|
||||
cfg := yip.YipConfig{
|
||||
Name: "K3s Kairos Cluster Provider",
|
||||
Stages: map[string][]yip.Stage{
|
||||
bootBefore: parseStages(cluster, parseFiles(cluster, systemName), systemName),
|
||||
},
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
func parseOptions(cluster clusterplugin.Cluster) ([]byte, []byte, []byte) {
|
||||
k3sConfig := &api.K3sServerConfig{
|
||||
Token: cluster.ClusterToken,
|
||||
}
|
||||
userOptionConfig := cluster.Options
|
||||
|
||||
switch cluster.Role {
|
||||
case clusterplugin.RoleInit:
|
||||
k3sConfig.ClusterInit = true
|
||||
k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
|
||||
case clusterplugin.RoleControlPlane:
|
||||
k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
|
||||
k3sConfig.TLSSan = []string{cluster.ControlPlaneHost}
|
||||
case clusterplugin.RoleWorker:
|
||||
userOptionConfig = ""
|
||||
k3sConfig.Server = fmt.Sprintf("https://%s:6443", cluster.ControlPlaneHost)
|
||||
// Data received from upstream contains config for both control plane and worker. Thus, for worker,
|
||||
// config is being filtered via unmarshal into agent config.
|
||||
var agentCfg api.K3sAgentConfig
|
||||
if err := yaml.Unmarshal([]byte(cluster.Options), &agentCfg); err == nil {
|
||||
out, _ := yaml.Marshal(agentCfg)
|
||||
userOptionConfig = string(out)
|
||||
} else {
|
||||
logrus.Fatalf("failed to un-marshal cluster options in k3s agent config %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
userOptions, _ := kyaml.YAMLToJSON([]byte(userOptionConfig))
|
||||
proxyOptions, _ := kyaml.YAMLToJSON([]byte(cluster.Options))
|
||||
options, _ := json.Marshal(k3sConfig)
|
||||
|
||||
// if provided, parse additional K3s server options (which may override the above settings)
|
||||
if cluster.ProviderOptions != nil && len(cluster.ProviderOptions) > 0 {
|
||||
logrus.Infof("applying cluster provider options: %+v", cluster.ProviderOptions)
|
||||
|
||||
providerOpts, err := yaml.Marshal(cluster.ProviderOptions)
|
||||
if err != nil {
|
||||
logrus.Fatalf("failed to marshal cluster.ProviderOptions: %v", err)
|
||||
}
|
||||
if err := yaml.Unmarshal(providerOpts, k3sConfig); err != nil {
|
||||
logrus.Fatalf("failed to unmarshal cluster.ProviderOptions: %v", err)
|
||||
}
|
||||
options, _ = json.Marshal(k3sConfig)
|
||||
|
||||
if v, ok := cluster.ProviderOptions[constants.ClusterInit]; ok && v == "no" {
|
||||
// Manually set cluster-init to false, as it's dropped by the above marshal.
|
||||
// We want to omit this field in all other scenarios, hence this special, ugly case.
|
||||
override := []byte(`{"cluster-init":false,`)
|
||||
options = append(override, options[1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
return options, proxyOptions, userOptions
|
||||
}
|
||||
|
||||
func parseFiles(cluster clusterplugin.Cluster, systemName string) []yip.File {
|
||||
options, proxyOptions, userOptions := parseOptions(cluster)
|
||||
|
||||
files := []yip.File{
|
||||
{
|
||||
Path: filepath.Join(configurationPath, "90_userdata.yaml"),
|
||||
Permissions: 0400,
|
||||
Content: string(userOptions),
|
||||
},
|
||||
{
|
||||
Path: filepath.Join(configurationPath, "99_userdata.yaml"),
|
||||
Permissions: 0400,
|
||||
Content: string(options),
|
||||
},
|
||||
}
|
||||
|
||||
proxyValues := proxyEnv(proxyOptions, cluster.Env)
|
||||
|
||||
if len(proxyValues) > 0 {
|
||||
logrus.Infof("setting proxy values %s", proxyValues)
|
||||
files = append(files, yip.File{
|
||||
Path: filepath.Join(containerdEnvConfigPath, systemName),
|
||||
Permissions: 0400,
|
||||
Content: proxyValues,
|
||||
})
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
func parseStages(cluster clusterplugin.Cluster, files []yip.File, systemName string) []yip.Stage {
|
||||
var stages []yip.Stage
|
||||
|
||||
stages = append(stages, yip.Stage{
|
||||
Name: constants.InstallK3sConfigFiles,
|
||||
Files: files,
|
||||
Commands: []string{
|
||||
fmt.Sprintf("jq -s 'def flatten: reduce .[] as $i([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end); [.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value)' %s/*.yaml > /etc/rancher/k3s/config.yaml", configurationPath),
|
||||
},
|
||||
})
|
||||
|
||||
if cluster.ImportLocalImages {
|
||||
if cluster.LocalImagesPath == "" {
|
||||
cluster.LocalImagesPath = localImagesPath
|
||||
}
|
||||
importStage := yip.Stage{
|
||||
Name: constants.ImportK3sImages,
|
||||
Commands: []string{
|
||||
"chmod +x /opt/k3s/scripts/import.sh",
|
||||
fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath),
|
||||
},
|
||||
}
|
||||
stages = append(stages, importStage)
|
||||
}
|
||||
|
||||
stages = append(stages,
|
||||
yip.Stage{
|
||||
Name: constants.EnableOpenRCServices,
|
||||
If: "[ -x /sbin/openrc-run ]",
|
||||
Commands: []string{
|
||||
fmt.Sprintf("rc-update add %s default >/dev/null", systemName),
|
||||
fmt.Sprintf("service %s start", systemName),
|
||||
},
|
||||
},
|
||||
yip.Stage{
|
||||
Name: constants.EnableSystemdServices,
|
||||
If: "[ -x /bin/systemctl ]",
|
||||
Commands: []string{
|
||||
fmt.Sprintf("systemctl enable %s", systemName),
|
||||
fmt.Sprintf("systemctl restart %s", systemName),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return stages
|
||||
}
|
||||
|
||||
func proxyEnv(proxyOptions []byte, proxyMap map[string]string) string {
|
||||
var proxy []string
|
||||
var noProxy string
|
||||
var isProxyConfigured bool
|
||||
|
||||
httpProxy := proxyMap["HTTP_PROXY"]
|
||||
httpsProxy := proxyMap["HTTPS_PROXY"]
|
||||
userNoProxy := proxyMap["NO_PROXY"]
|
||||
|
||||
defaultNoProxy := getDefaultNoProxy(proxyOptions)
|
||||
logrus.Infof("setting default no proxy to %s", defaultNoProxy)
|
||||
|
||||
if len(httpProxy) > 0 {
|
||||
proxy = append(proxy, fmt.Sprintf("HTTP_PROXY=%s", httpProxy))
|
||||
proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTP_PROXY=%s", httpProxy))
|
||||
isProxyConfigured = true
|
||||
}
|
||||
|
||||
if len(httpsProxy) > 0 {
|
||||
proxy = append(proxy, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy))
|
||||
proxy = append(proxy, fmt.Sprintf("CONTAINERD_HTTPS_PROXY=%s", httpsProxy))
|
||||
isProxyConfigured = true
|
||||
}
|
||||
|
||||
if isProxyConfigured {
|
||||
noProxy = defaultNoProxy
|
||||
}
|
||||
|
||||
if len(userNoProxy) > 0 {
|
||||
noProxy = noProxy + "," + userNoProxy
|
||||
}
|
||||
|
||||
if len(noProxy) > 0 {
|
||||
proxy = append(proxy, fmt.Sprintf("NO_PROXY=%s", noProxy))
|
||||
proxy = append(proxy, fmt.Sprintf("CONTAINERD_NO_PROXY=%s", noProxy))
|
||||
}
|
||||
|
||||
return strings.Join(proxy, "\n")
|
||||
}
|
||||
|
||||
func getDefaultNoProxy(proxyOptions []byte) string {
|
||||
var noProxy string
|
||||
|
||||
data := make(map[string]interface{})
|
||||
err := json.Unmarshal(proxyOptions, &data)
|
||||
if err != nil {
|
||||
logrus.Fatalf("error while unmarshalling user options %s", err)
|
||||
}
|
||||
|
||||
if data != nil {
|
||||
clusterCIDR := data["cluster-cidr"].(string)
|
||||
serviceCIDR := data["service-cidr"].(string)
|
||||
|
||||
if len(clusterCIDR) > 0 {
|
||||
noProxy = noProxy + "," + clusterCIDR
|
||||
}
|
||||
if len(serviceCIDR) > 0 {
|
||||
noProxy = noProxy + "," + serviceCIDR
|
||||
}
|
||||
}
|
||||
noProxy = noProxy + "," + getNodeCIDR() + "," + k8sNoProxy
|
||||
|
||||
return noProxy
|
||||
}
|
||||
|
||||
func getNodeCIDR() string {
|
||||
addrs, _ := net.InterfaceAddrs()
|
||||
var result string
|
||||
for _, addr := range addrs {
|
||||
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To4() != nil {
|
||||
result = addr.String()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
88
pkg/provider/provider_test.go
Normal file
88
pkg/provider/provider_test.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/kairos-io/kairos-sdk/clusterplugin"
|
||||
)
|
||||
|
||||
func Test_parseOptions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cluster clusterplugin.Cluster
|
||||
expectedOptions []byte
|
||||
expectedProxyOptions []byte
|
||||
expectedUserOptions []byte
|
||||
}{
|
||||
{
|
||||
name: "Empty input",
|
||||
cluster: clusterplugin.Cluster{},
|
||||
expectedOptions: []byte(`{}`),
|
||||
expectedProxyOptions: []byte(`null`),
|
||||
expectedUserOptions: []byte(`null`),
|
||||
},
|
||||
{
|
||||
name: "Init: Standard",
|
||||
cluster: clusterplugin.Cluster{
|
||||
ClusterToken: "token",
|
||||
ControlPlaneHost: "localhost",
|
||||
Role: "init",
|
||||
},
|
||||
expectedOptions: []byte(`{"cluster-init":true,"token":"token","tls-san":["localhost"]}`),
|
||||
expectedProxyOptions: []byte(`null`),
|
||||
expectedUserOptions: []byte(`null`),
|
||||
},
|
||||
{
|
||||
name: "Init: 2-Node",
|
||||
cluster: clusterplugin.Cluster{
|
||||
ClusterToken: "token",
|
||||
ControlPlaneHost: "localhost",
|
||||
Role: "init",
|
||||
ProviderOptions: map[string]string{
|
||||
"cluster-init": "no",
|
||||
"datastore-endpoint": "localhost:2379",
|
||||
},
|
||||
},
|
||||
expectedOptions: []byte(`{"cluster-init":false,"token":"token","tls-san":["localhost"],"datastore-endpoint":"localhost:2379"}`),
|
||||
expectedProxyOptions: []byte(`null`),
|
||||
expectedUserOptions: []byte(`null`),
|
||||
},
|
||||
{
|
||||
name: "Control Plane",
|
||||
cluster: clusterplugin.Cluster{
|
||||
ClusterToken: "token",
|
||||
ControlPlaneHost: "localhost",
|
||||
Role: "controlplane",
|
||||
},
|
||||
expectedOptions: []byte(`{"token":"token","server":"https://localhost:6443","tls-san":["localhost"]}`),
|
||||
expectedProxyOptions: []byte(`null`),
|
||||
expectedUserOptions: []byte(`null`),
|
||||
},
|
||||
{
|
||||
name: "Worker",
|
||||
cluster: clusterplugin.Cluster{
|
||||
ClusterToken: "token",
|
||||
ControlPlaneHost: "localhost",
|
||||
Role: "worker",
|
||||
},
|
||||
expectedOptions: []byte(`{"token":"token","server":"https://localhost:6443"}`),
|
||||
expectedProxyOptions: []byte(`null`),
|
||||
expectedUserOptions: []byte(`{}`),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
options, proxyOptions, userOptions := parseOptions(tt.cluster)
|
||||
if !bytes.Equal(options, tt.expectedOptions) {
|
||||
t.Errorf("parseOptions() options = %v, want %v", string(options), string(tt.expectedOptions))
|
||||
}
|
||||
if !bytes.Equal(proxyOptions, tt.expectedProxyOptions) {
|
||||
t.Errorf("parseOptions() proxyOptions = %v, want %v", string(proxyOptions), string(tt.expectedProxyOptions))
|
||||
}
|
||||
if !bytes.Equal(userOptions, tt.expectedUserOptions) {
|
||||
t.Errorf("parseOptions() userOptions = %v, want %v", string(userOptions), string(tt.expectedUserOptions))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user