Refactor, add dependant ops

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/deniswernert/go-fstab v0.0.0-20141204152952-eb4090f26517
 	github.com/kairos-io/kairos v1.5.0
 	github.com/moby/sys/mountinfo v0.5.0
-	github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c
+	github.com/spectrocloud-labs/herd v0.2.1
 	github.com/urfave/cli v1.22.10
 )
 

go.sum

@@ -637,6 +637,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c h1:l6aDSoktdHpSTsUEMZy6tkn0iMf/8Buzwh3DIkkWeoE=
 github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c/go.mod h1:fcZ8fjFcEJUM7qF6YcgxF3z8CCLjJeF7r4K1m8JQbHs=
+github.com/spectrocloud-labs/herd v0.2.1 h1:Z08zjr8i+DFD6hB/51FnMSDIf2Q3nKGtA25UMYJwKMI=
+github.com/spectrocloud-labs/herd v0.2.1/go.mod h1:fcZ8fjFcEJUM7qF6YcgxF3z8CCLjJeF7r4K1m8JQbHs=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

pkg/mount/fs.go

@@ -0,0 +1,186 @@
+package mount
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/deniswernert/go-fstab"
+	"github.com/kairos-io/immucore/pkg/profile"
+	"github.com/kairos-io/kairos/pkg/utils"
+	"github.com/moby/sys/mountinfo"
+)
+
+func rootFSType(s string) string {
+	out, _ := utils.SH(fmt.Sprintf("findmnt -rno FSTYPE %s", s))
+	return out
+}
+func createIfNotExists(path string) error {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return os.MkdirAll(path, os.ModePerm)
+	}
+
+	return nil
+}
+
+func appendSlash(path string) string {
+	if !strings.HasSuffix(path, "/") {
+		return fmt.Sprintf("%s/", path)
+	}
+
+	return path
+}
+
+// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L129
+func baseOverlay(overlay profile.Overlay) (mountOperation, error) {
+	if err := os.MkdirAll(overlay.Base, 0700); err != nil {
+		return mountOperation{}, err
+	}
+
+	dat := strings.Split(overlay.BackingBase, ":")
+
+	if len(dat) != 2 {
+		return mountOperation{}, fmt.Errorf("invalid backing base. must be a tmpfs with a size or a block device. e.g. tmpfs:30%%, block:/dev/sda1. Input: %s", overlay.BackingBase)
+	}
+
+	t := dat[0]
+	switch t {
+	case "tmpfs":
+		tmpMount := mount.Mount{Type: "tmpfs", Source: "tmpfs", Options: []string{"defaults", fmt.Sprintf("size=%s", dat[1])}}
+		err := mount.All([]mount.Mount{tmpMount}, overlay.Base)
+		fstab := mountToStab(tmpMount)
+		fstab.File = overlay.BackingBase
+		return mountOperation{
+			MountOption: tmpMount,
+			FstabEntry:  *fstab,
+			Target:      overlay.Base,
+		}, err
+	case "block":
+		blockMount := mount.Mount{Type: "auto", Source: dat[1]}
+		err := mount.All([]mount.Mount{blockMount}, overlay.Base)
+
+		fstab := mountToStab(blockMount)
+		fstab.File = overlay.BackingBase
+		fstab.MntOps["default"] = ""
+
+		return mountOperation{
+			MountOption: blockMount,
+			FstabEntry:  *fstab,
+			Target:      overlay.Base,
+		}, err
+	default:
+		return mountOperation{}, fmt.Errorf("invalid overlay backing base type")
+	}
+}
+
+func mountToStab(m mount.Mount) *fstab.Mount {
+	opts := map[string]string{}
+	for _, o := range m.Options {
+		if strings.Contains(o, "=") {
+			dat := strings.Split(o, "=")
+			key := dat[0]
+			value := dat[1]
+			opts[key] = value
+		} else {
+			opts[o] = ""
+		}
+	}
+	return &fstab.Mount{
+		Spec:    m.Source,
+		VfsType: m.Type,
+		MntOps:  opts,
+		Freq:    0,
+		PassNo:  0,
+	}
+}
+
+// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L183
+func mountBind(mountpoint, root, stateTarget string) (mountOperation, error) {
+	mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs
+	rootMount := filepath.Join(root, mountpoint)
+	bindMountPath := strings.ReplaceAll(mountpoint, "/", "-")
+
+	stateDir := filepath.Join(root, stateTarget, fmt.Sprintf("%s.bind", bindMountPath))
+
+	if mounted, _ := mountinfo.Mounted(rootMount); !mounted {
+		tmpMount := mount.Mount{
+			Type:   "overlay",
+			Source: stateDir,
+			Options: []string{
+				"defaults",
+				"bind",
+			},
+		}
+
+		fstab := mountToStab(tmpMount)
+		fstab.File = fmt.Sprintf("/%s", mountpoint)
+		fstab.Spec = strings.ReplaceAll(fstab.Spec, root, "")
+		return mountOperation{
+			MountOption: tmpMount,
+			FstabEntry:  *fstab,
+			Target:      rootMount,
+			PrepareCallback: func() error {
+				if err := createIfNotExists(rootMount); err != nil {
+					return err
+				}
+
+				if err := createIfNotExists(stateDir); err != nil {
+					return err
+				}
+
+				return syncState(appendSlash(rootMount), appendSlash(stateDir))
+			},
+		}, nil
+	}
+	return mountOperation{}, fmt.Errorf("already mounted")
+}
+
+func syncState(src, dst string) error {
+	return exec.Command("rsync", "-aqAX", src, dst).Run()
+}
+
+// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L145
+func mountWithBaseOverlay(mountpoint, root, base string) (mountOperation, error) {
+	mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs
+	rootMount := filepath.Join(root, mountpoint)
+	bindMountPath := strings.ReplaceAll(mountpoint, "/", "-")
+
+	createIfNotExists(rootMount)
+	if mounted, _ := mountinfo.Mounted(rootMount); !mounted {
+		upperdir := filepath.Join(base, bindMountPath, ".overlay", "upper")
+		workdir := filepath.Join(base, bindMountPath, ".overlay", "work")
+
+		tmpMount := mount.Mount{
+			Type:   "overlay",
+			Source: "overlay",
+			Options: []string{
+				"defaults",
+				fmt.Sprintf("lowerdir=%s", rootMount),
+				fmt.Sprintf("upperdir=%s", upperdir),
+				fmt.Sprintf("workdir=%s", workdir),
+			},
+		}
+
+		fstab := mountToStab(tmpMount)
+		fstab.File = rootMount
+
+		// TODO: update fstab with x-systemd info
+		// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L170
+		return mountOperation{
+			MountOption: tmpMount,
+			FstabEntry:  *fstab,
+			Target:      rootMount,
+			PrepareCallback: func() error {
+				// Make sure workdir and/or upper exists
+				os.MkdirAll(upperdir, os.ModePerm)
+				os.MkdirAll(workdir, os.ModePerm)
+				return nil
+			},
+		}, nil
+	}
+
+	return mountOperation{}, fmt.Errorf("already mounted")
+}

pkg/mount/mount.go

@@ -4,215 +4,22 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/containerd/containerd/mount"
 	"github.com/deniswernert/go-fstab"
 	"github.com/kairos-io/immucore/pkg/profile"
 	"github.com/kairos-io/kairos/pkg/utils"
-	"github.com/moby/sys/mountinfo"
 	"github.com/spectrocloud-labs/herd"
 )
 
-type MountOperation struct {
-	FstabEntry      fstab.Mount
-	MountOption     mount.Mount
-	Target          string
-	PrepareCallback func() error
-}
-
-func (m MountOperation) Run() error {
-	if m.PrepareCallback != nil {
-		if err := m.PrepareCallback(); err != nil {
-			return err
-		}
-	}
-	return mount.All([]mount.Mount{m.MountOption}, m.Target)
-}
-
-// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L129
-func BaseOverlay(overlay profile.Overlay) (MountOperation, error) {
-	if err := os.MkdirAll(overlay.Base, 0700); err != nil {
-		return MountOperation{}, err
-	}
-
-	dat := strings.Split(overlay.BackingBase, ":")
-
-	if len(dat) != 2 {
-		return MountOperation{}, fmt.Errorf("invalid backing base. must be a tmpfs with a size or a block device. e.g. tmpfs:30%%, block:/dev/sda1. Input: %s", overlay.BackingBase)
-	}
-
-	t := dat[0]
-	switch t {
-	case "tmpfs":
-		tmpMount := mount.Mount{Type: "tmpfs", Source: "tmpfs", Options: []string{"defaults", fmt.Sprintf("size=%s", dat[1])}}
-		err := mount.All([]mount.Mount{tmpMount}, overlay.Base)
-		fstab := mountToStab(tmpMount)
-		fstab.File = overlay.BackingBase
-		return MountOperation{
-			MountOption: tmpMount,
-			FstabEntry:  *fstab,
-			Target:      overlay.Base,
-		}, err
-	case "block":
-		blockMount := mount.Mount{Type: "auto", Source: dat[1]}
-		err := mount.All([]mount.Mount{blockMount}, overlay.Base)
-
-		fstab := mountToStab(blockMount)
-		fstab.File = overlay.BackingBase
-		fstab.MntOps["default"] = ""
-
-		return MountOperation{
-			MountOption: blockMount,
-			FstabEntry:  *fstab,
-			Target:      overlay.Base,
-		}, err
-	default:
-		return MountOperation{}, fmt.Errorf("invalid overlay backing base type")
-	}
-}
-
-func mountToStab(m mount.Mount) *fstab.Mount {
-	opts := map[string]string{}
-	for _, o := range m.Options {
-		if strings.Contains(o, "=") {
-			dat := strings.Split(o, "=")
-			key := dat[0]
-			value := dat[1]
-			opts[key] = value
-		} else {
-			opts[o] = ""
-		}
-	}
-	return &fstab.Mount{
-		Spec:    m.Source,
-		VfsType: m.Type,
-		MntOps:  opts,
-		Freq:    0,
-		PassNo:  0,
-	}
-}
-
-func MountEphemeral(path []string) {
-
-}
-
-func MountPeristentPaths() {
-
-}
-
-func createIfNotExists(path string) error {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return os.MkdirAll(path, os.ModePerm)
-	}
-
-	return nil
-}
-
-func appendSlash(path string) string {
-	if !strings.HasSuffix(path, "/") {
-		return fmt.Sprintf("%s/", path)
-	}
-
-	return path
-}
-
-// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L183
-func mountBind(mountpoint, root, stateTarget string) (MountOperation, error) {
-	mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs
-	rootMount := filepath.Join(root, mountpoint)
-	bindMountPath := strings.ReplaceAll(mountpoint, "/", "-")
-
-	stateDir := filepath.Join(root, stateTarget, fmt.Sprintf("%s.bind", bindMountPath))
-
-	if mounted, _ := mountinfo.Mounted(rootMount); !mounted {
-		tmpMount := mount.Mount{
-			Type:   "overlay",
-			Source: stateDir,
-			Options: []string{
-				"defaults",
-				"bind",
-			},
-		}
-
-		fstab := mountToStab(tmpMount)
-		fstab.File = fmt.Sprintf("/%s", mountpoint)
-		fstab.Spec = strings.ReplaceAll(fstab.Spec, root, "")
-		return MountOperation{
-			MountOption: tmpMount,
-			FstabEntry:  *fstab,
-			Target:      rootMount,
-			PrepareCallback: func() error {
-				if err := createIfNotExists(rootMount); err != nil {
-					return err
-				}
-
-				if err := createIfNotExists(stateDir); err != nil {
-					return err
-				}
-
-				return syncState(appendSlash(rootMount), appendSlash(stateDir))
-			},
-		}, nil
-	}
-	return MountOperation{}, fmt.Errorf("already mounted")
-}
-
-func syncState(src, dst string) error {
-	return exec.Command("rsync", "-aqAX", src, dst).Run()
-}
-
-// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L145
-func mountWithBaseOverlay(mountpoint, root, base string) (MountOperation, error) {
-	mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs
-	rootMount := filepath.Join(root, mountpoint)
-	bindMountPath := strings.ReplaceAll(mountpoint, "/", "-")
-
-	createIfNotExists(rootMount)
-	if mounted, _ := mountinfo.Mounted(rootMount); !mounted {
-		upperdir := filepath.Join(base, bindMountPath, ".overlay", "upper")
-		workdir := filepath.Join(base, bindMountPath, ".overlay", "work")
-
-		tmpMount := mount.Mount{
-			Type:   "overlay",
-			Source: "overlay",
-			Options: []string{
-				"defaults",
-				fmt.Sprintf("lowerdir=%s", rootMount),
-				fmt.Sprintf("upperdir=%s", upperdir),
-				fmt.Sprintf("workdir=%s", workdir),
-			},
-		}
-
-		fstab := mountToStab(tmpMount)
-		fstab.File = rootMount
-
-		// TODO: update fstab with x-systemd info
-		// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L170
-		return MountOperation{
-			MountOption: tmpMount,
-			FstabEntry:  *fstab,
-			Target:      rootMount,
-			PrepareCallback: func() error {
-				// Make sure workdir and/or upper exists
-				os.MkdirAll(upperdir, os.ModePerm)
-				os.MkdirAll(workdir, os.ModePerm)
-				return nil
-			},
-		}, nil
-	}
-
-	return MountOperation{}, fmt.Errorf("already mounted")
-}
-
 type State struct {
 	Rootdir     string
 	TargetImage string   // e.g. /cOS/active.img
 	OverlayDir  []string // e.g. /var
 	BindMounts  []string // e.g. /etc/kubernetes
+	StateDir    string   // e.g. "/usr/local/.state"
 
 	CustomMounts map[string]string // e.g. diskid : mountpoint
 
@@ -268,10 +75,12 @@ func (s *State) Register(g *herd.Graph) error {
 		),
 	)
 
+	// overlay mount start
+	if rootFSType(s.Rootdir) != "overlay" {
 		g.Add("mount-overlay-base",
 			herd.WithCallback(
 				func(ctx context.Context) error {
-				op, err := BaseOverlay(profile.Overlay{
+					op, err := baseOverlay(profile.Overlay{
 						Base:        "/run/overlay",
 						BackingBase: "tmpfs:20%",
 					})
@@ -279,15 +88,19 @@ func (s *State) Register(g *herd.Graph) error {
 						return err
 					}
 					s.fstabs = append(s.fstabs, &op.FstabEntry)
-				return op.Run()
+					return op.run()
 				},
 			),
 		)
+	}
+
+	overlayCondition := herd.ConditionalOption(func() bool { return rootFSType(s.Rootdir) != "overlay" }, herd.WithDeps("mount-overlay-base"))
 
 	// TODO: Add fsck
 	// mount overlay
 	for _, p := range s.OverlayDir {
 		g.Add("mount-overlays-base",
+			overlayCondition,
 			herd.WithCallback(
 				func(ctx context.Context) error {
 					op, err := mountWithBaseOverlay(p, s.Rootdir, "/run/overlay")
@@ -295,7 +108,7 @@ func (s *State) Register(g *herd.Graph) error {
 						return err
 					}
 					s.fstabs = append(s.fstabs, &op.FstabEntry)
-					return op.Run()
+					return op.run()
 				},
 			),
 		)
@@ -305,6 +118,7 @@ func (s *State) Register(g *herd.Graph) error {
 	for id, mountpoint := range s.CustomMounts {
 		g.Add(
 			genOpreferenceName(opCustomMounts, mountpoint),
+			overlayCondition,
 			herd.WithCallback(
 				s.MountOP(
 					id,
@@ -322,24 +136,28 @@ func (s *State) Register(g *herd.Graph) error {
 	for _, p := range s.BindMounts {
 		g.Add(
 			genOpreferenceName("mount-state", p),
+			overlayCondition,
 			herd.WithDeps(genOpreferenceFromMap(opCustomMounts, s.CustomMounts)...),
 			herd.WithCallback(
 				func(ctx context.Context) error {
-					op, err := mountBind(p, s.Rootdir, "/usr/local/.state")
+					op, err := mountBind(p, s.Rootdir, s.StateDir)
 					if err != nil {
 						return err
 					}
 					s.fstabs = append(s.fstabs, &op.FstabEntry)
-					return op.Run()
+					return op.run()
 				},
 			),
 		)
 	}
+
+	// overlay mount end
+
 	g.Add("mount-sysroot",
 		herd.WithCallback(
 			s.MountOP(
 				"/dev/disk/by-label/COS_ACTIVE",
-				s.path("/sysroot"),
+				s.Rootdir,
 				"auto",
 				[]string{
 					"ro", // or rw
@@ -357,7 +175,7 @@ func (s *State) Register(g *herd.Graph) error {
 		herd.WithCallback(
 			s.MountOP(
 				"/dev/disk/by-label/COS_OEM",
-				"/oem",
+				s.path("/oem"),
 				"auto",
 				[]string{
 					"rw",
@@ -414,13 +232,13 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du
 				}
 				fstab := mountToStab(mountPoint)
 				fstab.File = where
-				op := MountOperation{
+				op := mountOperation{
 					MountOption: mountPoint,
 					FstabEntry:  *fstab,
 					Target:      where,
 				}
 
-				err := op.Run()
+				err := op.run()
 				if err != nil {
 					continue
 				}

pkg/mount/operation.go

@@ -0,0 +1,22 @@
+package mount
+
+import (
+	"github.com/containerd/containerd/mount"
+	"github.com/deniswernert/go-fstab"
+)
+
+type mountOperation struct {
+	FstabEntry      fstab.Mount
+	MountOption     mount.Mount
+	Target          string
+	PrepareCallback func() error
+}
+
+func (m mountOperation) run() error {
+	if m.PrepareCallback != nil {
+		if err := m.PrepareCallback(); err != nil {
+			return err
+		}
+	}
+	return mount.All([]mount.Mount{m.MountOption}, m.Target)
+}