diff --git a/go.mod b/go.mod index 62fa257..5343816 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/deniswernert/go-fstab v0.0.0-20141204152952-eb4090f26517 github.com/kairos-io/kairos v1.5.0 github.com/moby/sys/mountinfo v0.5.0 - github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c + github.com/spectrocloud-labs/herd v0.2.1 github.com/urfave/cli v1.22.10 ) diff --git a/go.sum b/go.sum index 23e2871..9ca6c63 100644 --- a/go.sum +++ b/go.sum @@ -637,6 +637,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c h1:l6aDSoktdHpSTsUEMZy6tkn0iMf/8Buzwh3DIkkWeoE= github.com/spectrocloud-labs/herd v0.1.2-0.20230201084705-8e524743ec9c/go.mod h1:fcZ8fjFcEJUM7qF6YcgxF3z8CCLjJeF7r4K1m8JQbHs= +github.com/spectrocloud-labs/herd v0.2.1 h1:Z08zjr8i+DFD6hB/51FnMSDIf2Q3nKGtA25UMYJwKMI= +github.com/spectrocloud-labs/herd v0.2.1/go.mod h1:fcZ8fjFcEJUM7qF6YcgxF3z8CCLjJeF7r4K1m8JQbHs= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= diff --git a/pkg/mount/fs.go b/pkg/mount/fs.go new file mode 100644 index 0000000..e2b7d24 --- /dev/null +++ b/pkg/mount/fs.go @@ -0,0 +1,186 @@ +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/containerd/containerd/mount" + "github.com/deniswernert/go-fstab" + "github.com/kairos-io/immucore/pkg/profile" + "github.com/kairos-io/kairos/pkg/utils" + "github.com/moby/sys/mountinfo" +) + +func rootFSType(s string) string { + out, _ := utils.SH(fmt.Sprintf("findmnt -rno FSTYPE %s", s)) + return out +} +func createIfNotExists(path string) error { + if _, err := os.Stat(path); os.IsNotExist(err) { + return os.MkdirAll(path, os.ModePerm) + } + + return nil +} + +func appendSlash(path string) string { + if !strings.HasSuffix(path, "/") { + return fmt.Sprintf("%s/", path) + } + + return path +} + +// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L129 +func baseOverlay(overlay profile.Overlay) (mountOperation, error) { + if err := os.MkdirAll(overlay.Base, 0700); err != nil { + return mountOperation{}, err + } + + dat := strings.Split(overlay.BackingBase, ":") + + if len(dat) != 2 { + return mountOperation{}, fmt.Errorf("invalid backing base. must be a tmpfs with a size or a block device. e.g. tmpfs:30%%, block:/dev/sda1. Input: %s", overlay.BackingBase) + } + + t := dat[0] + switch t { + case "tmpfs": + tmpMount := mount.Mount{Type: "tmpfs", Source: "tmpfs", Options: []string{"defaults", fmt.Sprintf("size=%s", dat[1])}} + err := mount.All([]mount.Mount{tmpMount}, overlay.Base) + fstab := mountToStab(tmpMount) + fstab.File = overlay.BackingBase + return mountOperation{ + MountOption: tmpMount, + FstabEntry: *fstab, + Target: overlay.Base, + }, err + case "block": + blockMount := mount.Mount{Type: "auto", Source: dat[1]} + err := mount.All([]mount.Mount{blockMount}, overlay.Base) + + fstab := mountToStab(blockMount) + fstab.File = overlay.BackingBase + fstab.MntOps["default"] = "" + + return mountOperation{ + MountOption: blockMount, + FstabEntry: *fstab, + Target: overlay.Base, + }, err + default: + return mountOperation{}, fmt.Errorf("invalid overlay backing base type") + } +} + +func mountToStab(m mount.Mount) *fstab.Mount { + opts := map[string]string{} + for _, o := range m.Options { + if strings.Contains(o, "=") { + dat := strings.Split(o, "=") + key := dat[0] + value := dat[1] + opts[key] = value + } else { + opts[o] = "" + } + } + return &fstab.Mount{ + Spec: m.Source, + VfsType: m.Type, + MntOps: opts, + Freq: 0, + PassNo: 0, + } +} + +// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L183 +func mountBind(mountpoint, root, stateTarget string) (mountOperation, error) { + mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs + rootMount := filepath.Join(root, mountpoint) + bindMountPath := strings.ReplaceAll(mountpoint, "/", "-") + + stateDir := filepath.Join(root, stateTarget, fmt.Sprintf("%s.bind", bindMountPath)) + + if mounted, _ := mountinfo.Mounted(rootMount); !mounted { + tmpMount := mount.Mount{ + Type: "overlay", + Source: stateDir, + Options: []string{ + "defaults", + "bind", + }, + } + + fstab := mountToStab(tmpMount) + fstab.File = fmt.Sprintf("/%s", mountpoint) + fstab.Spec = strings.ReplaceAll(fstab.Spec, root, "") + return mountOperation{ + MountOption: tmpMount, + FstabEntry: *fstab, + Target: rootMount, + PrepareCallback: func() error { + if err := createIfNotExists(rootMount); err != nil { + return err + } + + if err := createIfNotExists(stateDir); err != nil { + return err + } + + return syncState(appendSlash(rootMount), appendSlash(stateDir)) + }, + }, nil + } + return mountOperation{}, fmt.Errorf("already mounted") +} + +func syncState(src, dst string) error { + return exec.Command("rsync", "-aqAX", src, dst).Run() +} + +// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L145 +func mountWithBaseOverlay(mountpoint, root, base string) (mountOperation, error) { + mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs + rootMount := filepath.Join(root, mountpoint) + bindMountPath := strings.ReplaceAll(mountpoint, "/", "-") + + createIfNotExists(rootMount) + if mounted, _ := mountinfo.Mounted(rootMount); !mounted { + upperdir := filepath.Join(base, bindMountPath, ".overlay", "upper") + workdir := filepath.Join(base, bindMountPath, ".overlay", "work") + + tmpMount := mount.Mount{ + Type: "overlay", + Source: "overlay", + Options: []string{ + "defaults", + fmt.Sprintf("lowerdir=%s", rootMount), + fmt.Sprintf("upperdir=%s", upperdir), + fmt.Sprintf("workdir=%s", workdir), + }, + } + + fstab := mountToStab(tmpMount) + fstab.File = rootMount + + // TODO: update fstab with x-systemd info + // https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L170 + return mountOperation{ + MountOption: tmpMount, + FstabEntry: *fstab, + Target: rootMount, + PrepareCallback: func() error { + // Make sure workdir and/or upper exists + os.MkdirAll(upperdir, os.ModePerm) + os.MkdirAll(workdir, os.ModePerm) + return nil + }, + }, nil + } + + return mountOperation{}, fmt.Errorf("already mounted") +} diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 859e4a9..d711437 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -4,215 +4,22 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" - "strings" "time" "github.com/containerd/containerd/mount" "github.com/deniswernert/go-fstab" "github.com/kairos-io/immucore/pkg/profile" "github.com/kairos-io/kairos/pkg/utils" - "github.com/moby/sys/mountinfo" "github.com/spectrocloud-labs/herd" ) -type MountOperation struct { - FstabEntry fstab.Mount - MountOption mount.Mount - Target string - PrepareCallback func() error -} - -func (m MountOperation) Run() error { - if m.PrepareCallback != nil { - if err := m.PrepareCallback(); err != nil { - return err - } - } - return mount.All([]mount.Mount{m.MountOption}, m.Target) -} - -// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L129 -func BaseOverlay(overlay profile.Overlay) (MountOperation, error) { - if err := os.MkdirAll(overlay.Base, 0700); err != nil { - return MountOperation{}, err - } - - dat := strings.Split(overlay.BackingBase, ":") - - if len(dat) != 2 { - return MountOperation{}, fmt.Errorf("invalid backing base. must be a tmpfs with a size or a block device. e.g. tmpfs:30%%, block:/dev/sda1. Input: %s", overlay.BackingBase) - } - - t := dat[0] - switch t { - case "tmpfs": - tmpMount := mount.Mount{Type: "tmpfs", Source: "tmpfs", Options: []string{"defaults", fmt.Sprintf("size=%s", dat[1])}} - err := mount.All([]mount.Mount{tmpMount}, overlay.Base) - fstab := mountToStab(tmpMount) - fstab.File = overlay.BackingBase - return MountOperation{ - MountOption: tmpMount, - FstabEntry: *fstab, - Target: overlay.Base, - }, err - case "block": - blockMount := mount.Mount{Type: "auto", Source: dat[1]} - err := mount.All([]mount.Mount{blockMount}, overlay.Base) - - fstab := mountToStab(blockMount) - fstab.File = overlay.BackingBase - fstab.MntOps["default"] = "" - - return MountOperation{ - MountOption: blockMount, - FstabEntry: *fstab, - Target: overlay.Base, - }, err - default: - return MountOperation{}, fmt.Errorf("invalid overlay backing base type") - } -} - -func mountToStab(m mount.Mount) *fstab.Mount { - opts := map[string]string{} - for _, o := range m.Options { - if strings.Contains(o, "=") { - dat := strings.Split(o, "=") - key := dat[0] - value := dat[1] - opts[key] = value - } else { - opts[o] = "" - } - } - return &fstab.Mount{ - Spec: m.Source, - VfsType: m.Type, - MntOps: opts, - Freq: 0, - PassNo: 0, - } -} - -func MountEphemeral(path []string) { - -} - -func MountPeristentPaths() { - -} - -func createIfNotExists(path string) error { - if _, err := os.Stat(path); os.IsNotExist(err) { - return os.MkdirAll(path, os.ModePerm) - } - - return nil -} - -func appendSlash(path string) string { - if !strings.HasSuffix(path, "/") { - return fmt.Sprintf("%s/", path) - } - - return path -} - -// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L183 -func mountBind(mountpoint, root, stateTarget string) (MountOperation, error) { - mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs - rootMount := filepath.Join(root, mountpoint) - bindMountPath := strings.ReplaceAll(mountpoint, "/", "-") - - stateDir := filepath.Join(root, stateTarget, fmt.Sprintf("%s.bind", bindMountPath)) - - if mounted, _ := mountinfo.Mounted(rootMount); !mounted { - tmpMount := mount.Mount{ - Type: "overlay", - Source: stateDir, - Options: []string{ - "defaults", - "bind", - }, - } - - fstab := mountToStab(tmpMount) - fstab.File = fmt.Sprintf("/%s", mountpoint) - fstab.Spec = strings.ReplaceAll(fstab.Spec, root, "") - return MountOperation{ - MountOption: tmpMount, - FstabEntry: *fstab, - Target: rootMount, - PrepareCallback: func() error { - if err := createIfNotExists(rootMount); err != nil { - return err - } - - if err := createIfNotExists(stateDir); err != nil { - return err - } - - return syncState(appendSlash(rootMount), appendSlash(stateDir)) - }, - }, nil - } - return MountOperation{}, fmt.Errorf("already mounted") -} - -func syncState(src, dst string) error { - return exec.Command("rsync", "-aqAX", src, dst).Run() -} - -// https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L145 -func mountWithBaseOverlay(mountpoint, root, base string) (MountOperation, error) { - mountpoint = strings.TrimLeft(mountpoint, "/") // normalize, remove / upfront as we are going to re-use it in subdirs - rootMount := filepath.Join(root, mountpoint) - bindMountPath := strings.ReplaceAll(mountpoint, "/", "-") - - createIfNotExists(rootMount) - if mounted, _ := mountinfo.Mounted(rootMount); !mounted { - upperdir := filepath.Join(base, bindMountPath, ".overlay", "upper") - workdir := filepath.Join(base, bindMountPath, ".overlay", "work") - - tmpMount := mount.Mount{ - Type: "overlay", - Source: "overlay", - Options: []string{ - "defaults", - fmt.Sprintf("lowerdir=%s", rootMount), - fmt.Sprintf("upperdir=%s", upperdir), - fmt.Sprintf("workdir=%s", workdir), - }, - } - - fstab := mountToStab(tmpMount) - fstab.File = rootMount - - // TODO: update fstab with x-systemd info - // https://github.com/kairos-io/packages/blob/94aa3bef3d1330cb6c6905ae164f5004b6a58b8c/packages/system/dracut/immutable-rootfs/30cos-immutable-rootfs/cos-mount-layout.sh#L170 - return MountOperation{ - MountOption: tmpMount, - FstabEntry: *fstab, - Target: rootMount, - PrepareCallback: func() error { - // Make sure workdir and/or upper exists - os.MkdirAll(upperdir, os.ModePerm) - os.MkdirAll(workdir, os.ModePerm) - return nil - }, - }, nil - } - - return MountOperation{}, fmt.Errorf("already mounted") -} - type State struct { Rootdir string TargetImage string // e.g. /cOS/active.img OverlayDir []string // e.g. /var BindMounts []string // e.g. /etc/kubernetes + StateDir string // e.g. "/usr/local/.state" CustomMounts map[string]string // e.g. diskid : mountpoint @@ -268,26 +75,32 @@ func (s *State) Register(g *herd.Graph) error { ), ) - g.Add("mount-overlay-base", - herd.WithCallback( - func(ctx context.Context) error { - op, err := BaseOverlay(profile.Overlay{ - Base: "/run/overlay", - BackingBase: "tmpfs:20%", - }) - if err != nil { - return err - } - s.fstabs = append(s.fstabs, &op.FstabEntry) - return op.Run() - }, - ), - ) + // overlay mount start + if rootFSType(s.Rootdir) != "overlay" { + g.Add("mount-overlay-base", + herd.WithCallback( + func(ctx context.Context) error { + op, err := baseOverlay(profile.Overlay{ + Base: "/run/overlay", + BackingBase: "tmpfs:20%", + }) + if err != nil { + return err + } + s.fstabs = append(s.fstabs, &op.FstabEntry) + return op.run() + }, + ), + ) + } + + overlayCondition := herd.ConditionalOption(func() bool { return rootFSType(s.Rootdir) != "overlay" }, herd.WithDeps("mount-overlay-base")) // TODO: Add fsck // mount overlay for _, p := range s.OverlayDir { g.Add("mount-overlays-base", + overlayCondition, herd.WithCallback( func(ctx context.Context) error { op, err := mountWithBaseOverlay(p, s.Rootdir, "/run/overlay") @@ -295,7 +108,7 @@ func (s *State) Register(g *herd.Graph) error { return err } s.fstabs = append(s.fstabs, &op.FstabEntry) - return op.Run() + return op.run() }, ), ) @@ -305,6 +118,7 @@ func (s *State) Register(g *herd.Graph) error { for id, mountpoint := range s.CustomMounts { g.Add( genOpreferenceName(opCustomMounts, mountpoint), + overlayCondition, herd.WithCallback( s.MountOP( id, @@ -322,24 +136,28 @@ func (s *State) Register(g *herd.Graph) error { for _, p := range s.BindMounts { g.Add( genOpreferenceName("mount-state", p), + overlayCondition, herd.WithDeps(genOpreferenceFromMap(opCustomMounts, s.CustomMounts)...), herd.WithCallback( func(ctx context.Context) error { - op, err := mountBind(p, s.Rootdir, "/usr/local/.state") + op, err := mountBind(p, s.Rootdir, s.StateDir) if err != nil { return err } s.fstabs = append(s.fstabs, &op.FstabEntry) - return op.Run() + return op.run() }, ), ) } + + // overlay mount end + g.Add("mount-sysroot", herd.WithCallback( s.MountOP( "/dev/disk/by-label/COS_ACTIVE", - s.path("/sysroot"), + s.Rootdir, "auto", []string{ "ro", // or rw @@ -357,7 +175,7 @@ func (s *State) Register(g *herd.Graph) error { herd.WithCallback( s.MountOP( "/dev/disk/by-label/COS_OEM", - "/oem", + s.path("/oem"), "auto", []string{ "rw", @@ -414,13 +232,13 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du } fstab := mountToStab(mountPoint) fstab.File = where - op := MountOperation{ + op := mountOperation{ MountOption: mountPoint, FstabEntry: *fstab, Target: where, } - err := op.Run() + err := op.run() if err != nil { continue } diff --git a/pkg/mount/operation.go b/pkg/mount/operation.go new file mode 100644 index 0000000..d88e518 --- /dev/null +++ b/pkg/mount/operation.go @@ -0,0 +1,22 @@ +package mount + +import ( + "github.com/containerd/containerd/mount" + "github.com/deniswernert/go-fstab" +) + +type mountOperation struct { + FstabEntry fstab.Mount + MountOption mount.Mount + Target string + PrepareCallback func() error +} + +func (m mountOperation) run() error { + if m.PrepareCallback != nil { + if err := m.PrepareCallback(); err != nil { + return err + } + } + return mount.All([]mount.Mount{m.MountOption}, m.Target) +}