PE-5119 Support cluster root path (#86)

* bump kairos sdk to v0.4.2

Signed-off-by: Nianyu Shen <xiaoyu9964@gmail.com>

* create bind mounts for k3s data,conf dir if cluste root path is provided and is not root

Signed-off-by: Nianyu Shen <xiaoyu9964@gmail.com>

* fix version

Signed-off-by: Nianyu Shen <xiaoyu9964@gmail.com>

* fix version

Signed-off-by: Nianyu Shen <xiaoyu9964@gmail.com>

* fix workflow

* fix: merge conflict

* add precommit config

* fix lint

---------

Signed-off-by: Nianyu Shen <xiaoyu9964@gmail.com>

.github/workflows/release.yaml

@@ -12,25 +12,33 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - uses: docker-practice/actions-setup-docker@master
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
       - uses: earthly/actions-setup@v1
         with:
-          version: "v0.6.30"
+          version: "latest"
       - run: earthly --ci +lint
   build-provider-package:
     runs-on: ubuntu-latest
     permissions:
       packages: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - uses: docker-practice/actions-setup-docker@master
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
       - uses: earthly/actions-setup@v1
         with:
-          version: "v0.6.30"
+          version: "latest"
-      - uses: docker/login-action@v1
+      - run: echo "${{ secrets.GCR_JSON_KEY_B64 }}" | base64 -d | docker login -u _json_key --password-stdin https://gcr.io
-        with:
+      - run: earthly --ci --push +provider-package-all-platforms --IMAGE_REPOSITORY=gcr.io/spectro-dev-public/kairos-io
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }} 
-      - run: earthly --ci --push +provider-package-all-platforms --IMAGE_REPOSITORY=ghcr.io/kairos-io

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+# install pre-commit on your system and then
+# run pre-commit install in this repository.
+# You can by pass commit hooks with:
+# git commit -n
+repos:
+  - repo: https://github.com/tekwizely/pre-commit-golang
+    rev: v1.0.0-rc.1
+    hooks:
+      - id: go-mod-tidy
+  - repo: https://github.com/golangci/golangci-lint
+    rev: v1.61.0
+    hooks:
+      - id: golangci-lint
+        name: golangci-lint
+        description: Fast linters runner for Go. Note that only modified files are linted, so linters like 'unused' that need to scan all files won't work as expected.
+        entry: golangci-lint run --new-from-rev HEAD --whole-files -v
+        types: [go]
+        language: golang
+        require_serial: true
+        pass_filenames: false
+        verbose: true

Earthfile

@@ -47,7 +47,7 @@ VERSION:
 
     COPY . ./
 
-    RUN echo $(git describe --exact-match --tags || echo "v0.0.0-$(git log --oneline -n 1 | cut -d" " -f1)") > VERSION
+    RUN echo $(git describe --exact-match --tags || echo "v0.0.0-$(git rev-parse --short=8 HEAD)") > VERSION
 
     SAVE ARTIFACT VERSION VERSION
 

go.mod

@@ -5,6 +5,7 @@ go 1.22.5
 require (
 	github.com/kairos-io/kairos-sdk v0.4.2
 	github.com/mudler/yip v1.9.4
+	github.com/onsi/gomega v1.34.2
 	github.com/sirupsen/logrus v1.9.3
 	gopkg.in/yaml.v3 v3.0.1
 	sigs.k8s.io/yaml v1.4.0
@@ -12,7 +13,7 @@ require (
 
 require (
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -20,12 +21,14 @@ require (
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/twpayne/go-vfs/v4 v4.3.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -54,8 +54,8 @@ github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5/go.mod h1:WmKc
 github.com/mudler/yip v1.9.4 h1:yaiPKWG5kt/DTNCf7ZGfyWdb1j5c06zYqWF3F+SVKsE=
 github.com/mudler/yip v1.9.4/go.mod h1:nqf8JFCq7a7rIkm7cSs+SOc8QbiyvVJ/xLbUw4GgzFs=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
@@ -136,7 +136,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

pkg/constants/constants.go

@@ -29,3 +29,8 @@ const (
 	// TLS key file used for client certificate based authentication to your datastore.
 	DatastoreKeyFile string = "datastore-keyfile"
 )
+
+const (
+	ClusterRootPath         = "cluster_root_path"
+	RunSystemdSystemDir = "/run/systemd/system"
+)

pkg/provider/mount.tmpl

@@ -0,0 +1,12 @@
+[Unit]
+Description={{.Name}} mount unit
+Before=local-fs.target k3s.service k3s-agent.service
+
+[Mount]
+What={{.Source}}
+Where={{.Target}}
+Type=none
+Options=bind
+
+[Install]
+WantedBy=local-fs.target

pkg/provider/provider.go

@@ -1,8 +1,10 @@
 package provider
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+
 	"net"
 	"path/filepath"
 	"strings"
@@ -13,10 +15,17 @@ import (
 	"gopkg.in/yaml.v3"
 	kyaml "sigs.k8s.io/yaml"
 
+	_ "embed"
+	"text/template"
+
 	"github.com/kairos-io/provider-k3s/api"
 	"github.com/kairos-io/provider-k3s/pkg/constants"
+	"github.com/kairos-io/provider-k3s/pkg/types"
 )
 
+//go:embed mount.tmpl
+var mountTemplate string
+
 const (
 	configurationPath       = "/etc/rancher/k3s/config.d"
 	containerdEnvConfigPath = "/etc/default"
@@ -81,7 +90,7 @@ func parseOptions(cluster clusterplugin.Cluster) ([]byte, []byte, []byte) {
 	options, _ := json.Marshal(k3sConfig)
 
 	// if provided, parse additional K3s server options (which may override the above settings)
-	if cluster.ProviderOptions != nil && len(cluster.ProviderOptions) > 0 {
+	if len(cluster.ProviderOptions) > 0 {
 		logrus.Infof("applying cluster provider options: %+v", cluster.ProviderOptions)
 
 		providerOpts, err := yaml.Marshal(cluster.ProviderOptions)
@@ -134,8 +143,54 @@ func parseFiles(cluster clusterplugin.Cluster, systemName string) []yip.File {
 	return files
 }
 
+func rootPathMountStage(rootPath string) yip.Stage {
+	mps := []types.MountPoint{
+		{
+			Name:   "etc-rancher",
+			Source: filepath.Join(rootPath, "etc/rancher"),
+			Target: "/etc/rancher",
+		},
+		{
+			Name:   "var-lib-rancher",
+			Source: filepath.Join(rootPath, "var/lib/rancher"),
+			Target: "/var/lib/rancher",
+		},
+	}
+
+	stage := yip.Stage{
+		Name: "Mount K3s data, conf directories",
+	}
+	for _, mp := range mps {
+		stage.Files = append(stage.Files, yip.File{
+			Path:        filepath.Join(constants.RunSystemdSystemDir, fmt.Sprintf("%s.mount", mp.Name)),
+			Permissions: 0644,
+			Content:     parseMountUnitFile(mp),
+		})
+
+		stage.Commands = append(stage.Commands,
+			fmt.Sprintf("mkdir -p %s", mp.Source),
+			fmt.Sprintf("mkdir -p %s", mp.Target),
+			fmt.Sprintf("systemctl enable --now %s.mount", mp.Name),
+		)
+	}
+
+	return stage
+}
+
+func parseMountUnitFile(mp types.MountPoint) string {
+	mount, _ := template.New("mount").Parse(mountTemplate)
+	var buf bytes.Buffer
+	_ = mount.Execute(&buf, mp)
+	return buf.String()
+}
+
 func parseStages(cluster clusterplugin.Cluster, files []yip.File, systemName string) []yip.Stage {
 	var stages []yip.Stage
+	clusterRootPath := getClusterRootPath(cluster)
+
+	if len(clusterRootPath) > 0 && clusterRootPath != "/" {
+		stages = append(stages, rootPathMountStage(clusterRootPath))
+	}
 
 	stages = append(stages, yip.Stage{
 		Name:  constants.InstallK3sConfigFiles,
@@ -152,8 +207,8 @@ func parseStages(cluster clusterplugin.Cluster, files []yip.File, systemName str
 		importStage := yip.Stage{
 			Name: constants.ImportK3sImages,
 			Commands: []string{
-				"chmod +x /opt/k3s/scripts/import.sh",
+				fmt.Sprintf("chmod +x %s/opt/k3s/scripts/import.sh", clusterRootPath),
-				fmt.Sprintf("/bin/sh /opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", cluster.LocalImagesPath),
+				fmt.Sprintf("/bin/sh %s/opt/k3s/scripts/import.sh %s > /var/log/k3s-import-images.log", clusterRootPath, filepath.Join(clusterRootPath, cluster.LocalImagesPath)),
 			},
 		}
 		stages = append(stages, importStage)
@@ -259,3 +314,7 @@ func getNodeCIDR() string {
 	}
 	return result
 }
+
+func getClusterRootPath(cluster clusterplugin.Cluster) string {
+	return cluster.ProviderOptions[constants.ClusterRootPath]
+}

pkg/provider/provider_test.go

@@ -2,9 +2,12 @@ package provider
 
 import (
 	"bytes"
+	_ "embed"
 	"testing"
 
 	"github.com/kairos-io/kairos-sdk/clusterplugin"
+	yip "github.com/mudler/yip/pkg/schema"
+	"github.com/onsi/gomega"
 )
 
 func Test_parseOptions(t *testing.T) {
@@ -86,3 +89,52 @@ func Test_parseOptions(t *testing.T) {
 		})
 	}
 }
+
+func Test_rootPathMountStage(t *testing.T) {
+	gomega.RegisterTestingT(t)
+
+	type args struct {
+		rootPath string
+	}
+	tests := []struct {
+		name string
+		args args
+		want yip.Stage
+	}{
+		{
+			name: "custom root path",
+			args: args{
+				rootPath: "/writable",
+			},
+			want: yip.Stage{
+				Name: "Mount K3s data, conf directories",
+				Commands: []string{
+					"mkdir -p /writable/etc/rancher",
+					"mkdir -p /etc/rancher",
+					"systemctl enable --now etc-rancher.mount",
+					"mkdir -p /writable/var/lib/rancher",
+					"mkdir -p /var/lib/rancher",
+					"systemctl enable --now var-lib-rancher.mount",
+				},
+				Files: []yip.File{
+					{
+						Path:        "/run/systemd/system/etc-rancher.mount",
+						Permissions: 0644,
+						Content:     "[Unit]\nDescription=etc-rancher mount unit\nBefore=local-fs.target k3s.service k3s-agent.service\n\n[Mount]\nWhat=/writable/etc/rancher\nWhere=/etc/rancher\nType=none\nOptions=bind\n\n[Install]\nWantedBy=local-fs.target\n",
+					},
+					{
+						Path:        "/run/systemd/system/var-lib-rancher.mount",
+						Permissions: 0644,
+						Content:     "[Unit]\nDescription=var-lib-rancher mount unit\nBefore=local-fs.target k3s.service k3s-agent.service\n\n[Mount]\nWhat=/writable/var/lib/rancher\nWhere=/var/lib/rancher\nType=none\nOptions=bind\n\n[Install]\nWantedBy=local-fs.target\n",
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := rootPathMountStage(tt.args.rootPath)
+			gomega.Expect(got).To(gomega.Equal(tt.want))
+		})
+	}
+}

pkg/types/types.go

@@ -0,0 +1,7 @@
+package types
+
+type MountPoint struct {
+	Name   string
+	Source string
+	Target string
+}