diff --git a/go.mod b/go.mod index dc4c6c6..534821a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/onsi/ginkgo/v2 v2.7.1 github.com/onsi/gomega v1.26.0 github.com/rs/zerolog v1.29.0 - github.com/spectrocloud-labs/herd v0.2.1 + github.com/spectrocloud-labs/herd v0.3.0 github.com/urfave/cli v1.22.10 ) @@ -28,6 +28,8 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/gookit/color v1.5.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/joho/godotenv v1.4.0 // indirect github.com/kendru/darwin/go/depgraph v0.0.0-20221105232959-877d6a81060c // indirect github.com/lithammer/fuzzysearch v1.1.5 // indirect diff --git a/go.sum b/go.sum index 322c786..31fb1c4 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,12 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -650,6 +654,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spectrocloud-labs/herd v0.2.1 h1:Z08zjr8i+DFD6hB/51FnMSDIf2Q3nKGtA25UMYJwKMI= github.com/spectrocloud-labs/herd v0.2.1/go.mod h1:fcZ8fjFcEJUM7qF6YcgxF3z8CCLjJeF7r4K1m8JQbHs= +github.com/spectrocloud-labs/herd v0.3.0 h1:n/VmHC/3NKfteYhiBonlFtohMRiMGuc6in0krqkyWMw= +github.com/spectrocloud-labs/herd v0.3.0/go.mod h1:RHPSzrH+Jd05+ewEpqk8ZgBgTsnHN8erkxwRdQAiw3Q= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 9a813e5..ee34d7a 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -5,58 +5,49 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/containerd/containerd/mount" "github.com/deniswernert/go-fstab" + "github.com/hashicorp/go-multierror" + "github.com/joho/godotenv" "github.com/kairos-io/immucore/pkg/profile" "github.com/kairos-io/kairos/pkg/utils" "github.com/spectrocloud-labs/herd" ) type State struct { - Rootdir string // e.g. /sysroot inside initrd with pivot, / with nopivot - TargetImage string // e.g. /cOS/active.img + Rootdir string // e.g. /sysroot inside initrd with pivot, / with nopivot + TargetImage string // e.g. /cOS/active.img + TargetLabel string // e.g. COS_ACTIVE + + // /run/cos-layout.env (different!) OverlayDir []string // e.g. /var BindMounts []string // e.g. /etc/kubernetes - StateDir string // e.g. "/usr/local/.state" - TargetLabel string // e.g. COS_ACTIVE - FStabFile string // e.g. /etc/fstab - MountRoot bool // e.g. if true, it tries to find the image to loopback mount CustomMounts map[string]string // e.g. diskid : mountpoint + StateDir string // e.g. "/usr/local/.state" + FStabFile string // e.g. /etc/fstab + MountRoot bool // e.g. if true, it tries to find the image to loopback mount + fstabs []*fstab.Mount } -func genOpreferenceName(op, s string) string { - return fmt.Sprintf("%s-%s", op, s) -} - -func genOpreferenceFromMap(op string, m map[string]string) (res []string) { - values := []string{} - for _, n := range m { - values = append(values, n) - } - - res = genOpreference(op, values) - return -} -func genOpreference(op string, s []string) (res []string) { - for _, n := range s { - res = append(res, genOpreferenceName(op, n)) - } - return -} - const ( - opCustomMounts = "custom-mount" - opDiscoverState = "discover-state" - opMountState = "mount-state" + opCustomMounts = "custom-mount" + opDiscoverState = "discover-state" + opMountState = "mount-state" + opMountBind = "mount-bind" + opMountRoot = "mount-root" opOverlayMount = "overlay-mount" opWriteFstab = "write-fstab" opMountBaseOverlay = "mount-base-overlay" opMountOEM = "mount-oem" + + opRootfsHook = "rootfs-hook" + opLoadConfig = "load-config" ) func (s *State) path(p ...string) string { @@ -84,6 +75,14 @@ func (s *State) WriteFstab(fstabFile string) func(context.Context) error { } } +// ln -sf -t / /sysroot/system +func (s *State) RunStageOp(stage string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := utils.SH(fmt.Sprintf("elemental run-stage %s", stage)) + return err + } +} + func (s *State) MountOP(what, where, t string, options []string, timeout time.Duration) func(context.Context) error { return func(c context.Context) error { for { @@ -134,12 +133,36 @@ func (s *State) WriteDAG(g *herd.Graph) (out string) { return } +func readEnv(file string) (map[string]string, error) { + var envMap map[string]string + var err error + + f, err := os.Open(file) + if err != nil { + return envMap, err + } + defer f.Close() + + envMap, err = godotenv.Parse(f) + if err != nil { + return envMap, err + } + + return envMap, err +} + func (s *State) Register(g *herd.Graph) error { - // TODO: add, hooks, fstab, systemd compat + // TODO: add hooks, fstab (might have missed some), systemd compat + // TODO: We should also set tmpfs here (not -related) + // symlink + // execute the rootfs hook + + // All of this below need to run after rootfs stage runs (so the layout file is created) // This is legacy - in UKI we don't need to found the img, this needs to run in a conditional if s.MountRoot { + // setup loopback mount for the image target for booting g.Add(opDiscoverState, herd.WithDeps(opMountState), herd.WithCallback( @@ -149,6 +172,7 @@ func (s *State) Register(g *herd.Graph) error { }, )) + // mount the state partition so to find the loopback device g.Add(opMountState, herd.WithCallback( s.MountOP( @@ -161,6 +185,7 @@ func (s *State) Register(g *herd.Graph) error { ), ) + // mount the loopback device as root of the fs g.Add(opMountRoot, herd.WithDeps(opDiscoverState), herd.WithCallback( @@ -182,8 +207,37 @@ func (s *State) Register(g *herd.Graph) error { } + // depending on /run/cos-layout.env + // This is building the mountRoot dependendency if it was enabled mountRootCondition := herd.ConditionalOption(func() bool { return s.MountRoot }, herd.WithDeps(opMountRoot)) + // TODO: this needs to be run after state is discovered + // TODO: add symlink if Rootdir != "" + // TODO: chroot? + g.Add(opRootfsHook, mountRootCondition, herd.WithDeps(opMountOEM), herd.WithCallback(s.RunStageOp("rootfs"))) + + // /run/cos-layout.env + // populate state bindmounts, overlaymounts, custommounts + g.Add(opLoadConfig, + herd.WithDeps(opRootfsHook), + herd.WithCallback(func(ctx context.Context) error { + + env, err := readEnv("/run/cos-layout.env") + if err != nil { + return err + } + + // populate from env here + s.OverlayDir = strings.Split(env["RW_PATHS"], " ") + + // TODO: PERSISTENT_STATE_TARGET /usr/local/.state + s.BindMounts = strings.Split(env["PERSISTENT_STATE_PATHS"], " ") + + // TODO: this needs to be parsed + // s.CustomMounts = strings.Split(env["VOLUMES"], " ") + return nil + })) + // end sysroot mount // overlay mount start @@ -206,65 +260,80 @@ func (s *State) Register(g *herd.Graph) error { } overlayCondition := herd.ConditionalOption(func() bool { return rootFSType(s.Rootdir) != "overlay" }, herd.WithDeps(opMountBaseOverlay)) - // TODO: Add fsck // mount overlay - for _, p := range s.OverlayDir { - g.Add( - genOpreferenceName(opOverlayMount, p), - overlayCondition, - mountRootCondition, - herd.WithCallback( - func(ctx context.Context) error { + + g.Add( + opOverlayMount, + overlayCondition, + herd.WithDeps(opLoadConfig), + mountRootCondition, + herd.WithCallback( + func(ctx context.Context) error { + var err error + for _, p := range s.OverlayDir { op, err := mountWithBaseOverlay(p, s.Rootdir, "/run/overlay") if err != nil { return err } s.fstabs = append(s.fstabs, &op.FstabEntry) - return op.run() - }, - ), - ) - } + err = multierror.Append(err, op.run()) + } - // custom mounts TODO: disk/path - for id, mountpoint := range s.CustomMounts { - g.Add( - genOpreferenceName(opCustomMounts, mountpoint), - mountRootCondition, - overlayCondition, - herd.WithCallback( - s.MountOP( + return err + }, + ), + ) + + g.Add( + opCustomMounts, + mountRootCondition, + overlayCondition, + herd.WithDeps(opLoadConfig), + herd.WithCallback(func(ctx context.Context) error { + var err error + + for id, mountpoint := range s.CustomMounts { + + err = multierror.Append(err, s.MountOP( id, s.path(mountpoint), "auto", []string{ "ro", // or rw - }, 60*time.Second), - ), - ) - } + }, + 60*time.Second, + )(ctx)) + + } + return err + }), + ) // mount state // mount state is defined over a custom mount (/usr/local/.state for instance, needs to be mounted over a device) - for _, p := range s.BindMounts { - g.Add( - genOpreferenceName(opMountState, p), - overlayCondition, - mountRootCondition, - herd.WithDeps(genOpreferenceFromMap(opCustomMounts, s.CustomMounts)...), - herd.WithCallback( - func(ctx context.Context) error { + g.Add( + opMountBind, + overlayCondition, + mountRootCondition, + herd.WithDeps(opCustomMounts, opLoadConfig), + herd.WithCallback( + func(ctx context.Context) error { + var err error + + for _, p := range s.BindMounts { + op, err := mountBind(p, s.Rootdir, s.StateDir) if err != nil { return err } s.fstabs = append(s.fstabs, &op.FstabEntry) - return op.run() - }, - ), - ) - } + err = multierror.Append(err, op.run()) + } + return err + }, + ), + ) // overlay mount end g.Add(opMountOEM, @@ -289,12 +358,9 @@ func (s *State) Register(g *herd.Graph) error { g.Add(opWriteFstab, overlayCondition, - herd.ConditionalOption(func() bool { return s.MountRoot }, herd.WithDeps(opMountRoot)), - herd.WithDeps(opMountOEM), + mountRootCondition, + herd.WithDeps(opMountOEM, opCustomMounts, opMountBind, opOverlayMount), herd.WeakDeps, - herd.WithDeps(genOpreferenceFromMap(opCustomMounts, s.CustomMounts)...), - herd.WithDeps(genOpreference(opMountState, s.BindMounts)...), - herd.WithDeps(genOpreference(opOverlayMount, s.OverlayDir)...), herd.WithCallback(s.WriteFstab(s.FStabFile))) return nil diff --git a/pkg/mount/mount_test.go b/pkg/mount/mount_test.go index fc95623..875e609 100644 --- a/pkg/mount/mount_test.go +++ b/pkg/mount/mount_test.go @@ -22,14 +22,20 @@ var _ = Describe("mounting immutable setup", func() { dag := g.Analyze() - Expect(len(dag)).To(Equal(3)) // Expect 3 layers - Expect(len(dag[0])).To(Equal(1)) // 1 Item for each layer, as are tight deps - Expect(len(dag[1])).To(Equal(1)) - Expect(len(dag[2])).To(Equal(1)) + Expect(len(dag)).To(Equal(7), s.WriteDAG(g)) // Expect 3 layers + Expect(len(dag[0])).To(Equal(1), s.WriteDAG(g)) // 1 Item for each layer, as are tight deps + Expect(len(dag[1])).To(Equal(1), s.WriteDAG(g)) + Expect(len(dag[2])).To(Equal(1), s.WriteDAG(g)) + Expect(len(dag[4])).To(Equal(2), s.WriteDAG(g)) - Expect(dag[0][0].Name).To(Equal("mount-base-overlay")) - Expect(dag[1][0].Name).To(Equal("mount-oem")) - Expect(dag[2][0].Name).To(Equal("write-fstab")) + Expect(dag[0][0].Name).To(Equal("mount-base-overlay"), s.WriteDAG(g)) + Expect(dag[1][0].Name).To(Equal("mount-oem"), s.WriteDAG(g)) + Expect(dag[2][0].Name).To(Equal("rootfs-hook"), s.WriteDAG(g)) + Expect(dag[3][0].Name).To(Equal("load-config"), s.WriteDAG(g)) + Expect(dag[4][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) + Expect(dag[4][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) + Expect(dag[5][0].Name).To(Equal("mount-bind"), s.WriteDAG(g)) + Expect(dag[6][0].Name).To(Equal("write-fstab"), s.WriteDAG(g)) }) It("mounts base overlay, attempt to mount oem, and updates the fstab", func() { @@ -39,49 +45,57 @@ var _ = Describe("mounting immutable setup", func() { dag := g.Analyze() - Expect(len(dag)).To(Equal(5), s.WriteDAG(g)) // Expect 4 layers + Expect(len(dag)).To(Equal(9), s.WriteDAG(g)) // Expect 4 layers Expect(len(dag[0])).To(Equal(2), s.WriteDAG(g)) // 2 items in first layer Expect(len(dag[1])).To(Equal(1)) // 1 Item for each layer, as are tight deps Expect(len(dag[2])).To(Equal(1)) Expect(len(dag[3])).To(Equal(1)) - Expect(len(dag[4])).To(Equal(1)) - - Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state"))) - Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state"))) + Expect(len(dag[6])).To(Equal(2)) + Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) + Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) Expect(dag[1][0].Name).To(Equal("discover-state")) Expect(dag[2][0].Name).To(Equal("mount-root")) Expect(dag[3][0].Name).To(Equal("mount-oem")) - Expect(dag[4][0].Name).To(Equal("write-fstab")) + Expect(dag[4][0].Name).To(Equal("rootfs-hook")) + Expect(dag[5][0].Name).To(Equal("load-config")) + Expect(dag[6][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) + Expect(dag[6][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) + + Expect(dag[7][0].Name).To(Equal("mount-bind")) + Expect(dag[8][0].Name).To(Equal("write-fstab")) + }) It("mounts all", func() { - s := &mount.State{Rootdir: "/", MountRoot: true, OverlayDir: []string{"/etc"}, BindMounts: []string{"/etc/kubernetes"}, CustomMounts: map[string]string{"COS_PERSISTENT": "/usr/local"}} + s := &mount.State{Rootdir: "/", MountRoot: true, + OverlayDir: []string{"/etc"}, + BindMounts: []string{"/etc/kubernetes"}, + CustomMounts: map[string]string{"COS_PERSISTENT": "/usr/local"}} s.Register(g) dag := g.Analyze() - Expect(len(dag)).To(Equal(6), s.WriteDAG(g)) // Expect 6 layers + Expect(len(dag)).To(Equal(9), s.WriteDAG(g)) // Expect 6 layers Expect(len(dag[0])).To(Equal(2), s.WriteDAG(g)) // 2 items in first layer - Expect(len(dag[1])).To(Equal(1), s.WriteDAG(g)) // 1 Item for each layer, as are tight deps - Expect(len(dag[2])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[3])).To(Equal(3), s.WriteDAG(g)) - Expect(len(dag[4])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[5])).To(Equal(1), s.WriteDAG(g)) - - Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state"))) - Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state"))) + Expect(len(dag[1])).To(Equal(1)) // 1 Item for each layer, as are tight deps + Expect(len(dag[2])).To(Equal(1)) + Expect(len(dag[3])).To(Equal(1)) + Expect(len(dag[6])).To(Equal(2)) + Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) + Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) Expect(dag[1][0].Name).To(Equal("discover-state")) Expect(dag[2][0].Name).To(Equal("mount-root")) + Expect(dag[3][0].Name).To(Equal("mount-oem")) + Expect(dag[4][0].Name).To(Equal("rootfs-hook")) + Expect(dag[5][0].Name).To(Equal("load-config")) + Expect(dag[6][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) + Expect(dag[6][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - Expect(dag[3][0].Name).To(Or(Equal("mount-oem"), Equal("overlay-mount-/etc"), Equal("custom-mount-/usr/local"))) - Expect(dag[3][1].Name).To(Or(Equal("mount-oem"), Equal("overlay-mount-/etc"), Equal("custom-mount-/usr/local"))) - Expect(dag[3][2].Name).To(Or(Equal("mount-oem"), Equal("overlay-mount-/etc"), Equal("custom-mount-/usr/local"))) - - Expect(dag[4][0].Name).To(Equal("mount-state-/etc/kubernetes")) - Expect(dag[5][0].Name).To(Equal("write-fstab")) + Expect(dag[7][0].Name).To(Equal("mount-bind")) + Expect(dag[8][0].Name).To(Equal("write-fstab")) }) }) })