From ad014e9f220ce16a6243f1214d1e9a4001def076 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 15 Feb 2023 22:30:08 +0100 Subject: [PATCH] Full rework (#41) * Full rework - Extract steps to a different file - Simplify dag for easy understanding - Load dag based on our boot process - Simplify steps to not depend on useless stuff - Better logging Signed-off-by: Itxaka * Move sentinel file to the dag Signed-off-by: Itxaka * Adapt tests Signed-off-by: Itxaka --------- Signed-off-by: Itxaka --- Earthfile | 2 +- dracut/28immucore/generator.sh | 2 +- go.mod | 3 + go.sum | 7 + internal/cmd/commands.go | 50 +- internal/constants/constants.go | 23 +- internal/utils/common.go | 44 -- pkg/mount/dag_live_media.go | 15 + pkg/mount/dag_normal_boot.go | 56 ++ pkg/mount/dag_steps.go | 349 ++++++++++++ pkg/mount/mount.go | 511 ------------------ pkg/mount/mount_test.go | 131 ----- pkg/mount/operation.go | 2 +- pkg/mount/state.go | 189 +++++++ ...ount_suite_test.go => state_suite_test.go} | 0 pkg/mount/state_test.go | 137 +++++ 16 files changed, 793 insertions(+), 728 deletions(-) create mode 100644 pkg/mount/dag_live_media.go create mode 100644 pkg/mount/dag_normal_boot.go create mode 100644 pkg/mount/dag_steps.go delete mode 100644 pkg/mount/mount.go delete mode 100644 pkg/mount/mount_test.go create mode 100644 pkg/mount/state.go rename pkg/mount/{mount_suite_test.go => state_suite_test.go} (100%) create mode 100644 pkg/mount/state_test.go diff --git a/Earthfile b/Earthfile index bb36175..7877899 100644 --- a/Earthfile +++ b/Earthfile @@ -59,7 +59,7 @@ build-immucore: COPY --dir pkg /work COPY +version/VERSION ./ ARG VERSION=$(cat VERSION) - ARG LDFLAGS="-s -w -X github.com/kairos-io/immucore/internal/version.Version=$VERSION" + ARG LDFLAGS="-s -w -X github.com/kairos-io/immucore/internal/version.version=$VERSION" RUN echo ${LDFLAGS} RUN CGO_ENABLED=0 go build -o immucore -ldflags "${LDFLAGS}" SAVE ARTIFACT /work/immucore AS LOCAL build/immucore-$VERSION diff --git a/dracut/28immucore/generator.sh b/dracut/28immucore/generator.sh index 5afca84..de90b9b 100755 --- a/dracut/28immucore/generator.sh +++ b/dracut/28immucore/generator.sh @@ -17,7 +17,7 @@ cos_img=$(getarg cos-img/filename=) # say, hey this is the ROOT where we need to boot! so it auto creates a sysroot.mount with the content of the value # passed in the cmdline. But because we usually pass the label of the img (COS_ACTIVE) it will create the wrong mount # service and be stuck in there forever. -# by generating it ourselves we get the sysroot.mount intot he generators.early dir, which tells systemd to not generate it +# by generating it ourselves we get the sysroot.mount into the generators.early dir, which tells systemd to not generate it # as it already exists and the rest is history { echo "[Unit]" diff --git a/go.mod b/go.mod index 643bf00..d93cbdb 100644 --- a/go.mod +++ b/go.mod @@ -30,9 +30,11 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gookit/color v1.5.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -61,6 +63,7 @@ require ( golang.org/x/sys v0.4.0 // indirect golang.org/x/term v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect + golang.org/x/tools v0.5.0 // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 98fd568..3fe185c 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= @@ -391,6 +393,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -432,6 +436,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -1006,6 +1011,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/cmd/commands.go b/internal/cmd/commands.go index 3c1eca2..b7a41ae 100644 --- a/internal/cmd/commands.go +++ b/internal/cmd/commands.go @@ -39,38 +39,22 @@ Sends a generic event payload with the configuration found in the scanned direct zerolog.SetGlobalLevel(zerolog.DebugLevel) } + g := herd.DAG(herd.EnableInit) + // You can pass rd.cos.disable in the cmdline to disable the whole immutable stuff - if len(utils.ReadCMDLineArg("rd.cos.disable")) > 0 { - log.Logger.Info().Msg("Stanza rd.cos.disable on the cmdline. Doing nothing.") - return nil - } - - // First set the sentinel file. - if !c.Bool("dry-run") { - err = utils.SetSentinelFile() - if err != nil { - log.Logger.Err(err).Send() - return err - } - } - - cdBoot, err := utils.BootedFromLiveMedia() - if err != nil { - log.Logger.Err(err).Send() - return err - } + cosDisable := len(utils.ReadCMDLineArg("rd.cos.disable")) > 0 img := utils.ReadCMDLineArg("cos-img/filename=") if len(img) == 0 { // If we boot from LIVE media or are using dry-run, we use a fake img as we still want to do things - if c.Bool("dry-run") || cdBoot { + if c.Bool("dry-run") || cosDisable { img = []string{"fake"} } else { log.Logger.Fatal().Msg("Could not get the image name from cmdline (i.e. cos-img/filename=/cOS/active.img)") } } log.Debug().Strs("TargetImage", img).Msg("Target image") - g := herd.DAG(herd.EnableInit) + s := &mount.State{ Logger: log.Logger, Rootdir: utils.GetRootDir(), @@ -79,7 +63,13 @@ Sends a generic event payload with the configuration found in the scanned direct TargetImage: img[0], } - err = s.Register(g) + if cosDisable { + log.Logger.Info().Msg("Stanza rd.cos.disable on the cmdline.") + err = s.RegisterLiveMedia(g) + } else { + err = s.RegisterNormalBoot(g) + } + if err != nil { s.Logger.Err(err) return err @@ -88,7 +78,7 @@ Sends a generic event payload with the configuration found in the scanned direct log.Info().Msg(s.WriteDAG(g)) if c.Bool("dry-run") { - return err + return nil } err = g.Run(context.Background()) @@ -97,17 +87,3 @@ Sends a generic event payload with the configuration found in the scanned direct }, }, } - -func writeDag(d [][]herd.GraphEntry) { - for i, layer := range d { - log.Printf("%d.", (i + 1)) - for _, op := range layer { - if op.Error != nil { - log.Printf(" <%s> (error: %s) (background: %t) (weak: %t)", op.Name, op.Error.Error(), op.Background, op.WeakDeps) - } else { - log.Printf(" <%s> (background: %t) (weak: %t)", op.Name, op.Background, op.WeakDeps) - } - } - log.Print("") - } -} diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 347fe8a..2d4dea1 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -2,11 +2,30 @@ package constants import "errors" -const PersistentStateTarget = "/usr/local/.state" - func DefaultRWPaths() []string { // Default RW_PATHS to mount if there are none defined return []string{"/etc", "/root", "/home", "/opt", "/srv", "/usr/local", "/var"} } var ErrAlreadyMounted = errors.New("already mounted") + +const ( + OpCustomMounts = "custom-mount" + OpDiscoverState = "discover-state" + OpMountState = "mount-state" + OpMountBind = "mount-bind" + + OpMountRoot = "mount-root" + OpOverlayMount = "overlay-mount" + OpWriteFstab = "write-fstab" + OpMountBaseOverlay = "mount-base-overlay" + OpMountOEM = "mount-oem" + + OpRootfsHook = "rootfs-hook" + OpLoadConfig = "load-config" + OpMountTmpfs = "mount-tmpfs" + + OpSentinel = "create-sentinel" + + PersistentStateTarget = "/usr/local/.state" +) diff --git a/internal/utils/common.go b/internal/utils/common.go index b3e2885..79bd1bb 100644 --- a/internal/utils/common.go +++ b/internal/utils/common.go @@ -4,20 +4,9 @@ import ( "github.com/joho/godotenv" "github.com/kairos-io/kairos/sdk/state" "os" - "path/filepath" "strings" ) -// BootedFromLiveMedia tells us if we are currently running off LIVE media like cd/usb or netboot -func BootedFromLiveMedia() (bool, error) { - runtime, err := state.NewRuntime() - if err != nil { - return false, err - } - - return runtime.BootState == state.LiveCD, nil -} - // BootStateToLabel lets us know the label we need to mount sysroot on func BootStateToLabel() string { runtime, err := state.NewRuntime() @@ -119,36 +108,3 @@ func CleanupSlice(slice []string) []string { } return cleanSlice } - -// SetSentinelFile sets the sentinel file to identify the boot mode. -// This is used by several things to know in which state they are, for example cloud configs -func SetSentinelFile() error { - var sentinel string - - err := CreateIfNotExists("/run/cos/") - if err != nil { - return err - } - runtime, err := state.NewRuntime() - if err != nil { - return err - } - - switch runtime.BootState { - case state.Active: - sentinel = "active_mode" - case state.Passive: - sentinel = "passive_mode" - case state.Recovery: - sentinel = "recovery_mode" - case state.LiveCD: - sentinel = "live_mode" - default: - sentinel = string(state.Unknown) - } - err = os.WriteFile(filepath.Join("/run/cos/", sentinel), []byte("1"), os.ModePerm) - if err != nil { - return err - } - return nil -} diff --git a/pkg/mount/dag_live_media.go b/pkg/mount/dag_live_media.go new file mode 100644 index 0000000..eefbf48 --- /dev/null +++ b/pkg/mount/dag_live_media.go @@ -0,0 +1,15 @@ +package mount + +import ( + "github.com/spectrocloud-labs/herd" +) + +// RegisterLiveMedia registers the dag for booting from live media/netboot +// This mounts /tmp +func (s *State) RegisterLiveMedia(g *herd.Graph) error { + var err error + s.LogIfError(s.MountTmpfsDagStep(g), "tmpfs mount") + // Maybe LogIfErrorAndPanic ? If no sentinel, a lot of config files are not going to run + err = s.LogIfErrorAndReturn(s.WriteSentinelDagStep(g), "write sentinel") + return err +} diff --git a/pkg/mount/dag_normal_boot.go b/pkg/mount/dag_normal_boot.go new file mode 100644 index 0000000..ca1408a --- /dev/null +++ b/pkg/mount/dag_normal_boot.go @@ -0,0 +1,56 @@ +package mount + +import ( + cnst "github.com/kairos-io/immucore/internal/constants" + "github.com/spectrocloud-labs/herd" +) + +// RegisterNormalBoot registers a dag for a normal boot, where we want to mount all the pieces that make up the +// final system. This mounts root, oem, runs rootfs, loads the cos-layout.env file and mounts custom stuff from that file +// and finally writes the fstab. +// This is all done on initramfs, very early, and ends up pivoting to the final system, usually under /sysroot +func (s *State) RegisterNormalBoot(g *herd.Graph) error { + var err error + + // TODO: add hooks, fstab (might have missed some), systemd compat, fsck + // Maybe LogIfErrorAndPanic ? If no sentinel, a lot of config files are not going to run + if err = s.LogIfErrorAndReturn(s.WriteSentinelDagStep(g), "write sentinel"); err != nil { + return err + } + + s.LogIfError(s.MountTmpfsDagStep(g), "tmpfs mount") + + // Mount Root (COS_STATE or COS_RECOVERY and then the image active/passive/recovery under s.Rootdir) + s.LogIfError(s.MountRootDagStep(g), "running mount root stage") + + // Mount COS_OEM (After root as it mounts under s.Rootdir/oem) + s.LogIfError(s.MountOemDagStep(g, cnst.OpMountRoot), "oem mount") + + // Run yip stage rootfs. Requires root+oem+sentinel to be mounted + s.LogIfError(s.RootfsStageDagStep(g, cnst.OpMountRoot, cnst.OpMountOEM, cnst.OpSentinel), "running rootfs stage") + + // Populate state bind mounts, overlay mounts, custom-mounts from /run/cos/cos-layout.env + // Requires stage rootfs to have run, which usually creates the cos-layout.env file + s.LogIfError(s.LoadEnvLayoutDagStep(g), "loading cos-layout.env") + + // Mount base overlay under /run/overlay + s.LogIfError(s.MountBaseOverlayDagStep(g), "base overlay mount") + + // Note(Itxaka): This was a dependency for overlayMount, opCustomMounts and opMountBind steps + // But I don't see how the s.Rootdir could ever be an overlay as we mount COS_STATE on it + // overlayCondition := herd.ConditionalOption(func() bool { return internalUtils.DiskFSType(s.Rootdir) != "overlay" }, herd.WithDeps(opMountBaseOverlay)) + + // Mount custom overlays loaded from the /run/cos/cos-layout.env file + s.LogIfError(s.MountCustomOverlayDagStep(g), "custom overlays mount") + + s.LogIfError(s.MountCustomMountsDagStep(g), "custom mounts mount") + + // Mount custom binds loaded from the /run/cos/cos-layout.env file + // Depends on mount binds as that usually mounts COS_PERSISTENT + s.LogIfError(s.MountCustomBindsDagStep(g), "custom binds mount") + + // Write fstab file + s.LogIfError(s.WriteFstabDagStep(g), "write fstab") + + return err +} diff --git a/pkg/mount/dag_steps.go b/pkg/mount/dag_steps.go new file mode 100644 index 0000000..31de80d --- /dev/null +++ b/pkg/mount/dag_steps.go @@ -0,0 +1,349 @@ +package mount + +import ( + "context" + "errors" + "fmt" + "github.com/hashicorp/go-multierror" + cnst "github.com/kairos-io/immucore/internal/constants" + internalUtils "github.com/kairos-io/immucore/internal/utils" + "github.com/kairos-io/kairos/pkg/utils" + "github.com/kairos-io/kairos/sdk/state" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spectrocloud-labs/herd" + "os" + "path/filepath" + "strings" + "time" +) + +// MountTmpfsDagStep adds the step to mount /tmp +func (s *State) MountTmpfsDagStep(g *herd.Graph) error { + return g.Add(cnst.OpMountTmpfs, herd.WithCallback(s.MountOP("tmpfs", "/tmp", "tmpfs", []string{"rw"}, 10*time.Second))) +} + +// MountRootDagStep will add the step to mount the Rootdir for the system +// 1 - mount the state partition to find the images (active/passive/recovery) +// 2 - mount the image as a loop device +// 3 - Mount the labels as /sysroot +func (s *State) MountRootDagStep(g *herd.Graph) error { + var err error + runtime, err := state.NewRuntime() + if err != nil { + s.Logger.Debug().Err(err).Msg("runtime") + } + stateName := runtime.State.Name + stateFs := runtime.State.Type + // Recovery is a different partition + if internalUtils.IsRecovery() { + stateName = runtime.Recovery.Name + stateFs = runtime.Recovery.Type + } + // 1 - mount the state partition to find the images (active/passive/recovery) + err = g.Add(cnst.OpMountState, + herd.WithCallback( + s.MountOP( + stateName, + s.path("/run/initramfs/cos-state"), + stateFs, + []string{ + "ro", // or rw + }, 60*time.Second), + ), + ) + if err != nil { + s.Logger.Err(err).Send() + } + + // 2 - mount the image as a loop device + err = g.Add(cnst.OpDiscoverState, + herd.WithDeps(cnst.OpMountState), + herd.WithCallback( + func(ctx context.Context) error { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() + // Check if loop device is mounted by checking the existance of the target label + if internalUtils.IsMountedByLabel(s.TargetLabel) { + log.Logger.Debug().Str("targetImage", s.TargetImage).Str("path", s.Rootdir).Str("TargetLabel", s.TargetLabel).Msg("Not mounting loop, already mounted") + return nil + } + // TODO: squashfs recovery image? + cmd := fmt.Sprintf("losetup --show -f %s", s.path("/run/initramfs/cos-state", s.TargetImage)) + _, err := utils.SH(cmd) + if err != nil { + log.Logger.Debug().Err(err).Msg("") + } + return err + }, + )) + if err != nil { + s.Logger.Err(err).Send() + } + + // 3 - Mount the labels as Rootdir + err = g.Add(cnst.OpMountRoot, + herd.WithDeps(cnst.OpDiscoverState), + herd.WithCallback( + s.MountOP( + // Using /dev/disk/by-label here allows us to not have to deal with loop devices to identify where was the image mounted + fmt.Sprintf("/dev/disk/by-label/%s", s.TargetLabel), + s.Rootdir, + "ext4", // are images always ext2? + []string{ + "ro", // or rw + "suid", + "dev", + "exec", + // "auto", + //"nouser", + "async", + }, 10*time.Second), + ), + ) + if err != nil { + s.Logger.Err(err).Send() + } + return err +} + +// RootfsStageDagStep will add the rootfs stage. +func (s *State) RootfsStageDagStep(g *herd.Graph, deps ...string) error { + return g.Add(cnst.OpRootfsHook, herd.WithDeps(deps...), herd.WithCallback(s.RunStageOp("rootfs"))) +} + +// LoadEnvLayoutDagStep will add the stage to load from cos-layout.env and fill the proper CustomMounts, OverlayDirs and BindMounts +func (s *State) LoadEnvLayoutDagStep(g *herd.Graph) error { + return g.Add(cnst.OpLoadConfig, + herd.WithDeps(cnst.OpRootfsHook), + herd.WithCallback(func(ctx context.Context) error { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() + if s.CustomMounts == nil { + s.CustomMounts = map[string]string{} + } + + env, err := internalUtils.ReadEnv("/run/cos/cos-layout.env") + if err != nil { + log.Logger.Err(err).Msg("Reading env") + return err + } + // populate from env here + s.OverlayDirs = internalUtils.CleanupSlice(strings.Split(env["RW_PATHS"], " ")) + // Append default RW_Paths if Dirs are empty + if len(s.OverlayDirs) == 0 { + s.OverlayDirs = cnst.DefaultRWPaths() + } + // Remove any duplicates + s.OverlayDirs = internalUtils.UniqueSlice(s.OverlayDirs) + + s.BindMounts = internalUtils.CleanupSlice(strings.Split(env["PERSISTENT_STATE_PATHS"], " ")) + // Remove any duplicates + s.BindMounts = internalUtils.UniqueSlice(s.BindMounts) + + s.StateDir = env["PERSISTENT_STATE_TARGET"] + if s.StateDir == "" { + s.StateDir = cnst.PersistentStateTarget + } + + addLine := func(d string) { + dat := strings.Split(d, ":") + if len(dat) == 2 { + disk := dat[0] + path := dat[1] + s.CustomMounts[disk] = path + } + } + // Parse custom mounts also from cmdline (rd.cos.mount=) + // Parse custom mounts also from env file (VOLUMES) + for _, v := range append(internalUtils.ReadCMDLineArg("rd.cos.mount="), strings.Split(env["VOLUMES"], " ")...) { + addLine(internalUtils.ParseMount(v)) + } + + return nil + })) +} + +// MountOemDagStep will add mounting COS_OEM partition under s.Rootdir + /oem +func (s *State) MountOemDagStep(g *herd.Graph, deps ...string) error { + runtime, err := state.NewRuntime() + if err != nil { + s.Logger.Debug().Err(err).Msg("runtime") + } + return g.Add(cnst.OpMountOEM, + herd.WithDeps(deps...), + herd.WithCallback( + s.MountOP( + fmt.Sprintf("/dev/disk/by-label/%s", runtime.OEM.Label), + s.path("/oem"), + runtime.OEM.Type, + []string{ + "rw", + "suid", + "dev", + "exec", + //"noauto", + //"nouser", + "async", + }, 10*time.Second), + ), + ) +} + +// MountBaseOverlayDagStep will add mounting /run/overlay as an overlay dir +func (s *State) MountBaseOverlayDagStep(g *herd.Graph) error { + return g.Add(cnst.OpMountBaseOverlay, + herd.WithCallback( + func(ctx context.Context) error { + op, err := baseOverlay(Overlay{ + Base: "/run/overlay", + BackingBase: "tmpfs:20%", + }) + if err != nil { + return err + } + err2 := op.run() + // No error, add fstab + if err2 == nil { + s.fstabs = append(s.fstabs, &op.FstabEntry) + return nil + } + // Error but its already mounted error, dont add fstab but dont return error + if err2 != nil && errors.Is(err2, cnst.ErrAlreadyMounted) { + return nil + } + + return err2 + }, + ), + ) +} + +// MountCustomOverlayDagStep will add mounting s.OverlayDirs under /run/overlay +func (s *State) MountCustomOverlayDagStep(g *herd.Graph) error { + return g.Add(cnst.OpOverlayMount, + herd.WithDeps(cnst.OpLoadConfig), + herd.WithCallback( + func(ctx context.Context) error { + var multierr *multierror.Error + s.Logger.Debug().Strs("dirs", s.OverlayDirs).Msg("Mounting overlays") + for _, p := range s.OverlayDirs { + op := mountWithBaseOverlay(p, s.Rootdir, "/run/overlay") + err := op.run() + // Append to errors only if it's not an already mounted error + if err != nil && !errors.Is(err, cnst.ErrAlreadyMounted) { + log.Logger.Err(err).Msg("overlay mount") + multierr = multierror.Append(multierr, err) + continue + } + s.fstabs = append(s.fstabs, &op.FstabEntry) + } + return multierr.ErrorOrNil() + }, + ), + ) +} + +// MountCustomMountsDagStep will add mounting s.CustomMounts +func (s *State) MountCustomMountsDagStep(g *herd.Graph) error { + return g.Add(cnst.OpCustomMounts, + herd.WithDeps(cnst.OpLoadConfig), + herd.WithCallback(func(ctx context.Context) error { + var err *multierror.Error + + for what, where := range s.CustomMounts { + // TODO: scan for the custom mount disk to know the underlying fs and set it proper + fstype := "ext4" + mountOptions := []string{"ro"} + // Persistent needs to be RW + if strings.Contains(what, "COS_PERSISTENT") { + mountOptions = []string{"rw"} + } + err = multierror.Append(err, s.MountOP( + what, + s.path(where), + fstype, + mountOptions, + 10*time.Second, + )(ctx)) + + } + s.Logger.Err(err.ErrorOrNil()).Send() + + return err.ErrorOrNil() + }), + ) +} + +// MountCustomBindsDagStep will add mounting s.BindMounts +// mount state is defined over a custom mount (/usr/local/.state for instance, needs to be mounted over a device) +func (s *State) MountCustomBindsDagStep(g *herd.Graph) error { + return g.Add(cnst.OpMountBind, + herd.WithDeps(cnst.OpCustomMounts, cnst.OpLoadConfig), + herd.WithCallback( + func(ctx context.Context) error { + var err *multierror.Error + s.Logger.Debug().Strs("mounts", s.BindMounts).Msg("Mounting binds") + + for _, p := range s.BindMounts { + op := mountBind(p, s.Rootdir, s.StateDir) + err2 := op.run() + if err2 == nil { + // Only append to fstabs if there was no error, otherwise we will try to mount it after switch_root + s.fstabs = append(s.fstabs, &op.FstabEntry) + } + // Append to errors only if it's not an already mounted error + if err2 != nil && !errors.Is(err2, cnst.ErrAlreadyMounted) { + log.Logger.Err(err2).Send() + err = multierror.Append(err, err2) + } + } + log.Logger.Err(err.ErrorOrNil()).Send() + return err.ErrorOrNil() + }, + ), + ) +} + +// WriteFstabDagStep will add writing the final fstab file with all the mounts +// Depends on everything but weak, so it will still try to write +func (s *State) WriteFstabDagStep(g *herd.Graph) error { + return g.Add(cnst.OpWriteFstab, + herd.WithDeps(cnst.OpMountRoot, cnst.OpDiscoverState, cnst.OpLoadConfig, cnst.OpMountOEM, cnst.OpCustomMounts, cnst.OpMountBind, cnst.OpOverlayMount), + herd.WeakDeps, + herd.WithCallback(s.WriteFstab(s.path("/etc/fstab")))) +} + +// WriteSentinelDagStep sets the sentinel file to identify the boot mode. +// This is used by several things to know in which state they are, for example cloud configs +func (s *State) WriteSentinelDagStep(g *herd.Graph) error { + return g.Add(cnst.OpSentinel, + herd.WithCallback(func(ctx context.Context) error { + var sentinel string + + err := internalUtils.CreateIfNotExists("/run/cos/") + if err != nil { + return err + } + runtime, err := state.NewRuntime() + if err != nil { + return err + } + + switch runtime.BootState { + case state.Active: + sentinel = "active_mode" + case state.Passive: + sentinel = "passive_mode" + case state.Recovery: + sentinel = "recovery_mode" + case state.LiveCD: + sentinel = "live_mode" + default: + sentinel = string(state.Unknown) + } + err = os.WriteFile(filepath.Join("/run/cos/", sentinel), []byte("1"), os.ModePerm) + if err != nil { + return err + } + return nil + })) +} diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go deleted file mode 100644 index 108e286..0000000 --- a/pkg/mount/mount.go +++ /dev/null @@ -1,511 +0,0 @@ -package mount - -import ( - "context" - "errors" - "fmt" - "github.com/kairos-io/immucore/internal/constants" - "os" - "path/filepath" - "strings" - "time" - - "github.com/containerd/containerd/mount" - "github.com/deniswernert/go-fstab" - "github.com/hashicorp/go-multierror" - internalUtils "github.com/kairos-io/immucore/internal/utils" - "github.com/kairos-io/kairos/pkg/utils" - "github.com/kairos-io/kairos/sdk/state" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spectrocloud-labs/herd" -) - -type State struct { - Logger zerolog.Logger - Rootdir string // e.g. /sysroot inside initrd with pivot, / with nopivot - TargetImage string // e.g. /cOS/active.img - TargetLabel string // e.g. COS_ACTIVE - - // /run/cos-layout.env (different!) - OverlayDirs []string // e.g. /var - BindMounts []string // e.g. /etc/kubernetes - CustomMounts map[string]string // e.g. diskid : mountpoint - - StateDir string // e.g. "/usr/local/.state" - MountRoot bool // e.g. if true, it tries to find the image to loopback mount - - fstabs []*fstab.Mount -} - -const ( - opCustomMounts = "custom-mount" - opDiscoverState = "discover-state" - opMountState = "mount-state" - opMountBind = "mount-bind" - - opMountRoot = "mount-root" - opOverlayMount = "overlay-mount" - opWriteFstab = "write-fstab" - opMountBaseOverlay = "mount-base-overlay" - opMountOEM = "mount-oem" - - opRootfsHook = "rootfs-hook" - opLoadConfig = "load-config" - opMountTmpfs = "mount-tmpfs" -) - -func (s *State) path(p ...string) string { - return filepath.Join(append([]string{s.Rootdir}, p...)...) -} - -func (s *State) WriteFstab(fstabFile string) func(context.Context) error { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() - return func(ctx context.Context) error { - for _, fst := range s.fstabs { - select { - case <-ctx.Done(): - default: - f, err := os.OpenFile(fstabFile, - os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - if _, err := f.WriteString(fmt.Sprintf("%s\n", fst.String())); err != nil { - _ = f.Close() - return err - } - _ = f.Close() - } - } - return nil - } -} - -func (s *State) RunStageOp(stage string) func(context.Context) error { - return func(ctx context.Context) error { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() - if stage == "rootfs" { - if _, err := os.Stat("/system"); os.IsNotExist(err) { - err = os.Symlink("/sysroot/system", "/system") - if err != nil { - s.Logger.Err(err).Msg("creating symlink") - return err - } - } - if _, err := os.Stat("/oem"); os.IsNotExist(err) { - err = os.Symlink("/sysroot/oem", "/oem") - if err != nil { - s.Logger.Err(err).Msg("creating symlink") - return err - } - } - } - - cmd := fmt.Sprintf("elemental run-stage %s", stage) - // If we set the level to debug, also call elemental with debug - if log.Logger.GetLevel() == zerolog.DebugLevel { - cmd = fmt.Sprintf("%s --debug", cmd) - } - output, err := utils.SH(cmd) - log.Debug().Msg(output) - return err - } -} - -func (s *State) MountOP(what, where, t string, options []string, timeout time.Duration) func(context.Context) error { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger() - - return func(c context.Context) error { - cc := time.After(timeout) - for { - select { - default: - err := internalUtils.CreateIfNotExists(where) - if err != nil { - log.Logger.Err(err).Msg("Creating dir") - continue - } - time.Sleep(1 * time.Second) - mountPoint := mount.Mount{ - Type: t, - Source: what, - Options: options, - } - tmpFstab := internalUtils.MountToFstab(mountPoint) - tmpFstab.File = internalUtils.CleanSysrootForFstab(where) - op := mountOperation{ - MountOption: mountPoint, - FstabEntry: *tmpFstab, - Target: where, - } - - err = op.run() - - if err == nil { - s.fstabs = append(s.fstabs, tmpFstab) - } - - // only continue the loop if it's an error and not an already mounted error - if err != nil && !errors.Is(err, constants.ErrAlreadyMounted) { - continue - } - - return nil - case <-c.Done(): - e := fmt.Errorf("context canceled") - log.Logger.Err(e).Msg("mount canceled") - return e - case <-cc: - e := fmt.Errorf("timeout exhausted") - log.Logger.Err(e).Msg("Mount timeout") - return e - } - } - } -} - -func (s *State) WriteDAG(g *herd.Graph) (out string) { - for i, layer := range g.Analyze() { - out += fmt.Sprintf("%d.\n", i+1) - for _, op := range layer { - if op.Error != nil { - out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps) - } else { - out += fmt.Sprintf(" <%s> (background: %t) (weak: %t)\n", op.Name, op.Background, op.WeakDeps) - } - } - } - return -} - -func (s *State) Register(g *herd.Graph) error { - var err error - - runtime, err := state.NewRuntime() - if err != nil { - s.Logger.Debug().Err(err).Msg("runtime") - return err - } - - s.Logger.Debug().Interface("runtime", runtime).Msg("Current runtime") - - // TODO: add hooks, fstab (might have missed some), systemd compat - // TODO: We should also set tmpfs here (not -related) - err = g.Add(opMountTmpfs, herd.WithCallback(s.MountOP("tmpfs", "/tmp", "tmpfs", []string{"rw"}, 10*time.Second))) - if err != nil { - s.Logger.Debug().Err(err).Msg("tmpfs mount") - return err - } - - if runtime.BootState == state.LiveCD { - s.Logger.Info().Msg("Booting from LiveCD") - return nil - } - - // All of this below need to run after rootfs stage runs (so the layout file is created) - // This is legacy - in UKI we don't need to found the img, this needs to run in a conditional - if s.MountRoot { - // setup loopback mount for the image target for booting - err = g.Add(opDiscoverState, - herd.WithDeps(opMountState), - herd.WithCallback( - func(ctx context.Context) error { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() - // Check if loop device is mounted by checking the existance of the target label - if internalUtils.IsMountedByLabel(s.TargetLabel) { - log.Logger.Debug().Str("targetImage", s.TargetImage).Str("path", s.Rootdir).Str("TargetLabel", s.TargetLabel).Msg("Not mounting loop, already mounted") - return nil - } - // TODO: squashfs recovery image? - cmd := fmt.Sprintf("losetup --show -f %s", s.path("/run/initramfs/cos-state", s.TargetImage)) - _, err := utils.SH(cmd) - if err != nil { - log.Logger.Debug().Err(err).Msg("") - } - return err - }, - )) - if err != nil { - s.Logger.Err(err).Send() - } - - // mount the state partition so to find the loopback device - stateName := runtime.State.Name - stateFs := runtime.State.Type - // Recovery is a different partition - if internalUtils.IsRecovery() { - stateName = runtime.Recovery.Name - stateFs = runtime.Recovery.Type - } - err = g.Add(opMountState, - herd.WithCallback( - s.MountOP( - stateName, - s.path("/run/initramfs/cos-state"), - stateFs, - []string{ - "ro", // or rw - }, 60*time.Second), - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - - // mount the loopback device as root of the fs - err = g.Add(opMountRoot, - herd.WithDeps(opDiscoverState), - herd.WithCallback( - s.MountOP( - // Using /dev/disk/by-label here allows us to not have to deal with loop devices to identify where was the image mounted - fmt.Sprintf("/dev/disk/by-label/%s", s.TargetLabel), - s.Rootdir, - "ext4", // are images always ext2? - []string{ - "ro", // or rw - "suid", - "dev", - "exec", - // "auto", - //"nouser", - "async", - }, 10*time.Second), - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - - } - - // depending on /run/cos-layout.env - // This is building the mountRoot dependendency if it was enabled - mountRootCondition := herd.ConditionalOption(func() bool { return s.MountRoot }, herd.WithDeps(opMountRoot)) - - // this needs to be run after sysroot, so we can link to /sysroot/system/oem and after /oem mounted - err = g.Add(opRootfsHook, mountRootCondition, herd.WithDeps(opMountRoot, opMountOEM), herd.WithCallback(s.RunStageOp("rootfs"))) - if err != nil { - s.Logger.Err(err).Msg("running rootfs stage") - } - - // /run/cos/cos-layout.env - // populate state bindmounts, overlaymounts, custommounts - err = g.Add(opLoadConfig, - herd.WithDeps(opRootfsHook), - herd.WithCallback(func(ctx context.Context) error { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() - if s.CustomMounts == nil { - s.CustomMounts = map[string]string{} - } - - env, err := internalUtils.ReadEnv("/run/cos/cos-layout.env") - if err != nil { - log.Logger.Err(err).Msg("Reading env") - return err - } - // populate from env here - s.OverlayDirs = internalUtils.CleanupSlice(strings.Split(env["RW_PATHS"], " ")) - // Append default RW_Paths if Dirs are empty - if len(s.OverlayDirs) == 0 { - s.OverlayDirs = constants.DefaultRWPaths() - } - // Remove any duplicates - s.OverlayDirs = internalUtils.UniqueSlice(s.OverlayDirs) - - s.BindMounts = internalUtils.CleanupSlice(strings.Split(env["PERSISTENT_STATE_PATHS"], " ")) - // Remove any duplicates - s.BindMounts = internalUtils.UniqueSlice(s.BindMounts) - - s.StateDir = env["PERSISTENT_STATE_TARGET"] - if s.StateDir == "" { - s.StateDir = constants.PersistentStateTarget - } - - addLine := func(d string) { - dat := strings.Split(d, ":") - if len(dat) == 2 { - disk := dat[0] - path := dat[1] - s.CustomMounts[disk] = path - } - } - // Parse custom mounts also from cmdline (rd.cos.mount=) - // Parse custom mounts also from env file (VOLUMES) - for _, v := range append(internalUtils.ReadCMDLineArg("rd.cos.mount="), strings.Split(env["VOLUMES"], " ")...) { - addLine(internalUtils.ParseMount(v)) - } - - return nil - })) - if err != nil { - s.Logger.Err(err).Send() - } - // end sysroot mount - - // overlay mount start - if internalUtils.DiskFSType(s.Rootdir) != "overlay" { - err = g.Add(opMountBaseOverlay, - herd.WithCallback( - func(ctx context.Context) error { - op, err := baseOverlay(Overlay{ - Base: "/run/overlay", - BackingBase: "tmpfs:20%", - }) - if err != nil { - return err - } - err2 := op.run() - // No error, add fstab - if err2 == nil { - s.fstabs = append(s.fstabs, &op.FstabEntry) - return nil - } - // Error but its already mounted error, dont add fstab but dont return error - if err2 != nil && errors.Is(err2, constants.ErrAlreadyMounted) { - return nil - } - - return err2 - }, - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - } - - overlayCondition := herd.ConditionalOption(func() bool { return internalUtils.DiskFSType(s.Rootdir) != "overlay" }, herd.WithDeps(opMountBaseOverlay)) - // TODO: Add fsck - // mount overlay - err = g.Add( - opOverlayMount, - overlayCondition, - herd.WithDeps(opLoadConfig), - mountRootCondition, - herd.WithCallback( - func(ctx context.Context) error { - var multierr *multierror.Error - s.Logger.Debug().Strs("dirs", s.OverlayDirs).Msg("Mounting overlays") - for _, p := range s.OverlayDirs { - op := mountWithBaseOverlay(p, s.Rootdir, "/run/overlay") - err := op.run() - // Append to errors only if it's not an already mounted error - if err != nil && !errors.Is(err, constants.ErrAlreadyMounted) { - log.Logger.Err(err).Msg("overlay mount") - multierr = multierror.Append(multierr, err) - continue - } - s.fstabs = append(s.fstabs, &op.FstabEntry) - } - return multierr.ErrorOrNil() - }, - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - err = g.Add( - opCustomMounts, - mountRootCondition, - overlayCondition, - herd.WithDeps(opLoadConfig), - herd.WithCallback(func(ctx context.Context) error { - var err *multierror.Error - - for what, where := range s.CustomMounts { - // TODO: scan for the custom mount disk to know the underlying fs and set it proper - fstype := "ext4" - mountOptions := []string{"ro"} - // Translate /disk/by-label/LABEL to disk for COS_PERSISTENT - // Persistent needs to be RW - if strings.Contains(what, "COS_PERSISTENT") { - what = runtime.Persistent.Name - fstype = runtime.Persistent.Type - mountOptions = []string{"rw"} - } - err = multierror.Append(err, s.MountOP( - what, - s.path(where), - fstype, - mountOptions, - 10*time.Second, - )(ctx)) - - } - s.Logger.Err(err.ErrorOrNil()).Send() - - return err.ErrorOrNil() - }), - ) - if err != nil { - s.Logger.Err(err).Send() - } - - // mount state is defined over a custom mount (/usr/local/.state for instance, needs to be mounted over a device) - err = g.Add( - opMountBind, - overlayCondition, - mountRootCondition, - herd.WithDeps(opCustomMounts, opLoadConfig), - herd.WithCallback( - func(ctx context.Context) error { - var err *multierror.Error - s.Logger.Debug().Strs("mounts", s.BindMounts).Msg("Mounting binds") - - for _, p := range s.BindMounts { - op := mountBind(p, s.Rootdir, s.StateDir) - err2 := op.run() - if err2 == nil { - // Only append to fstabs if there was no error, otherwise we will try to mount it after switch_root - s.fstabs = append(s.fstabs, &op.FstabEntry) - } - // Append to errors only if it's not an already mounted error - if err2 != nil && !errors.Is(err2, constants.ErrAlreadyMounted) { - log.Logger.Err(err2).Send() - err = multierror.Append(err, err2) - } - } - log.Logger.Err(err.ErrorOrNil()).Send() - return err.ErrorOrNil() - }, - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - - // overlay mount end - err = g.Add(opMountOEM, - overlayCondition, - mountRootCondition, - herd.WithCallback( - s.MountOP( - runtime.OEM.Name, - s.path("/oem"), - runtime.OEM.Type, - []string{ - "rw", - "suid", - "dev", - "exec", - //"noauto", - //"nouser", - "async", - }, 10*time.Second), - ), - ) - if err != nil { - s.Logger.Err(err).Send() - } - err = g.Add(opWriteFstab, - overlayCondition, - mountRootCondition, - herd.WithDeps(opMountOEM, opCustomMounts, opMountBind, opOverlayMount), - herd.WeakDeps, - herd.WithCallback(s.WriteFstab(s.path("/etc/fstab")))) - if err != nil { - s.Logger.Err(err).Send() - } - return err -} diff --git a/pkg/mount/mount_test.go b/pkg/mount/mount_test.go deleted file mode 100644 index 18a8c4c..0000000 --- a/pkg/mount/mount_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package mount_test - -import ( - "context" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "os" - "time" - - "github.com/kairos-io/immucore/pkg/mount" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/spectrocloud-labs/herd" -) - -var _ = Describe("mounting immutable setup", func() { - var g *herd.Graph - - BeforeEach(func() { - g = herd.DAG() - Expect(g).ToNot(BeNil()) - }) - - Context("simple invocation", func() { - It("mounts base overlay, attempt to mount oem, and updates the fstab", func() { - s := &mount.State{ - Logger: log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger(), - Rootdir: "/", - TargetImage: "/cOS/myimage.img", - TargetLabel: "COS_LABEL", - MountRoot: true, - } - - err := s.Register(g) - Expect(err).ToNot(HaveOccurred()) - - dag := g.Analyze() - - Expect(len(dag)).To(Equal(9), s.WriteDAG(g)) - Expect(len(dag[0])).To(Equal(2), s.WriteDAG(g)) - Expect(len(dag[1])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[2])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[3])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[4])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[5])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[6])).To(Equal(2), s.WriteDAG(g)) - Expect(len(dag[7])).To(Equal(1), s.WriteDAG(g)) - Expect(len(dag[8])).To(Equal(1), s.WriteDAG(g)) - - Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[1][0].Name).To(Equal("discover-state"), s.WriteDAG(g)) - Expect(dag[2][0].Name).To(Equal("mount-root"), s.WriteDAG(g)) - Expect(dag[3][0].Name).To(Equal("mount-oem"), s.WriteDAG(g)) - Expect(dag[4][0].Name).To(Equal("rootfs-hook"), s.WriteDAG(g)) - Expect(dag[5][0].Name).To(Equal("load-config"), s.WriteDAG(g)) - Expect(dag[6][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - Expect(dag[6][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - Expect(dag[7][0].Name).To(Equal("mount-bind"), s.WriteDAG(g)) - Expect(dag[8][0].Name).To(Equal("write-fstab"), s.WriteDAG(g)) - }) - - It("mounts base overlay, attempt to mount oem, and updates the fstab", func() { - s := &mount.State{Rootdir: "/", MountRoot: true} - - s.Register(g) - - dag := g.Analyze() - - Expect(len(dag)).To(Equal(9), s.WriteDAG(g)) // Expect 4 layers - Expect(len(dag[0])).To(Equal(2), s.WriteDAG(g)) // 2 items in first layer - Expect(len(dag[1])).To(Equal(1)) // 1 Item for each layer, as are tight deps - Expect(len(dag[2])).To(Equal(1)) - Expect(len(dag[3])).To(Equal(1)) - Expect(len(dag[6])).To(Equal(2)) - - Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[1][0].Name).To(Equal("discover-state")) - Expect(dag[2][0].Name).To(Equal("mount-root")) - Expect(dag[3][0].Name).To(Equal("mount-oem")) - Expect(dag[4][0].Name).To(Equal("rootfs-hook")) - Expect(dag[5][0].Name).To(Equal("load-config")) - Expect(dag[6][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - Expect(dag[6][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - - Expect(dag[7][0].Name).To(Equal("mount-bind")) - Expect(dag[8][0].Name).To(Equal("write-fstab")) - - }) - - It("mounts all", func() { - s := &mount.State{Rootdir: "/", MountRoot: true, - OverlayDirs: []string{"/etc"}, - BindMounts: []string{"/etc/kubernetes"}, - CustomMounts: map[string]string{"COS_PERSISTENT": "/usr/local"}} - - s.Register(g) - - dag := g.Analyze() - - Expect(len(dag)).To(Equal(9), s.WriteDAG(g)) // Expect 6 layers - Expect(len(dag[0])).To(Equal(2), s.WriteDAG(g)) // 2 items in first layer - Expect(len(dag[1])).To(Equal(1)) // 1 Item for each layer, as are tight deps - Expect(len(dag[2])).To(Equal(1)) - Expect(len(dag[3])).To(Equal(1)) - Expect(len(dag[6])).To(Equal(2)) - - Expect(dag[0][0].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[0][1].Name).To(Or(Equal("mount-base-overlay"), Equal("mount-state")), s.WriteDAG(g)) - Expect(dag[1][0].Name).To(Equal("discover-state")) - Expect(dag[2][0].Name).To(Equal("mount-root")) - Expect(dag[3][0].Name).To(Equal("mount-oem")) - Expect(dag[4][0].Name).To(Equal("rootfs-hook")) - Expect(dag[5][0].Name).To(Equal("load-config")) - Expect(dag[6][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - Expect(dag[6][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), s.WriteDAG(g)) - - Expect(dag[7][0].Name).To(Equal("mount-bind")) - Expect(dag[8][0].Name).To(Equal("write-fstab")) - }) - - It("Mountop timeouts", func() { - s := &mount.State{} - f := s.MountOP("/dev/doesntexist", "/tmp", "", []string{}, 1*time.Second) - err := f(context.Background()) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("exhausted")) - }) - }) -}) diff --git a/pkg/mount/operation.go b/pkg/mount/operation.go index a69419a..4d7bef1 100644 --- a/pkg/mount/operation.go +++ b/pkg/mount/operation.go @@ -35,6 +35,6 @@ func (m mountOperation) run() error { log.Logger.Debug().Msg("Already mounted") return constants.ErrAlreadyMounted } - log.Logger.Debug().Msg("Mounted") + log.Logger.Debug().Msg("mount ready") return mount.All([]mount.Mount{m.MountOption}, m.Target) } diff --git a/pkg/mount/state.go b/pkg/mount/state.go new file mode 100644 index 0000000..97422d8 --- /dev/null +++ b/pkg/mount/state.go @@ -0,0 +1,189 @@ +package mount + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/containerd/containerd/mount" + "github.com/deniswernert/go-fstab" + "github.com/kairos-io/immucore/internal/constants" + internalUtils "github.com/kairos-io/immucore/internal/utils" + "github.com/kairos-io/kairos/pkg/utils" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spectrocloud-labs/herd" +) + +type State struct { + Logger zerolog.Logger + Rootdir string // e.g. /sysroot inside initrd with pivot, / with nopivot + TargetImage string // e.g. /cOS/active.img + TargetLabel string // e.g. COS_ACTIVE + + // /run/cos-layout.env (different!) + OverlayDirs []string // e.g. /var + BindMounts []string // e.g. /etc/kubernetes + CustomMounts map[string]string // e.g. diskid : mountpoint + + StateDir string // e.g. "/usr/local/.state" + MountRoot bool // e.g. if true, it tries to find the image to loopback mount + fstabs []*fstab.Mount +} + +func (s *State) path(p ...string) string { + return filepath.Join(append([]string{s.Rootdir}, p...)...) +} + +func (s *State) WriteFstab(fstabFile string) func(context.Context) error { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger() + return func(ctx context.Context) error { + for _, fst := range s.fstabs { + select { + case <-ctx.Done(): + default: + f, err := os.OpenFile(fstabFile, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + if _, err := f.WriteString(fmt.Sprintf("%s\n", fst.String())); err != nil { + _ = f.Close() + return err + } + _ = f.Close() + } + } + return nil + } +} + +// RunStageOp runs elemental run-stage stage. If its rootfs its special as it needs som symlinks +func (s *State) RunStageOp(stage string) func(context.Context) error { + return func(ctx context.Context) error { + if stage == "rootfs" { + if _, err := os.Stat("/system"); os.IsNotExist(err) { + err = os.Symlink("/sysroot/system", "/system") + if err != nil { + s.Logger.Err(err).Msg("creating symlink") + } + } + if _, err := os.Stat("/oem"); os.IsNotExist(err) { + err = os.Symlink("/sysroot/oem", "/oem") + if err != nil { + s.Logger.Err(err).Msg("creating symlink") + } + } + } + + cmd := fmt.Sprintf("elemental run-stage %s", stage) + // If we set the level to debug, also call elemental with debug + if s.Logger.GetLevel() == zerolog.DebugLevel { + cmd = fmt.Sprintf("%s --debug", cmd) + } + output, err := utils.SH(cmd) + s.Logger.Debug().Msg(output) + return err + } +} + +// MountOP creates and executes a mount operation +func (s *State) MountOP(what, where, t string, options []string, timeout time.Duration) func(context.Context) error { + log.Logger.With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger() + + return func(c context.Context) error { + cc := time.After(timeout) + for { + select { + default: + err := internalUtils.CreateIfNotExists(where) + if err != nil { + log.Logger.Err(err).Msg("Creating dir") + continue + } + time.Sleep(1 * time.Second) + mountPoint := mount.Mount{ + Type: t, + Source: what, + Options: options, + } + tmpFstab := internalUtils.MountToFstab(mountPoint) + tmpFstab.File = internalUtils.CleanSysrootForFstab(where) + op := mountOperation{ + MountOption: mountPoint, + FstabEntry: *tmpFstab, + Target: where, + } + + err = op.run() + + if err == nil { + s.fstabs = append(s.fstabs, tmpFstab) + } + + // only continue the loop if it's an error and not an already mounted error + if err != nil && !errors.Is(err, constants.ErrAlreadyMounted) { + out, _ := utils.SH("blkid") + s.Logger.Debug().Msg(out) + continue + } + log.Logger.Debug().Msg("mount done") + return nil + case <-c.Done(): + e := fmt.Errorf("context canceled") + log.Logger.Err(e).Msg("mount canceled") + return e + case <-cc: + e := fmt.Errorf("timeout exhausted") + log.Logger.Err(e).Msg("Mount timeout") + return e + } + } + } +} + +// WriteDAG writes the dag +func (s *State) WriteDAG(g *herd.Graph) (out string) { + for i, layer := range g.Analyze() { + out += fmt.Sprintf("%d.\n", i+1) + for _, op := range layer { + if op.Error != nil { + out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps) + } else { + out += fmt.Sprintf(" <%s> (background: %t) (weak: %t)\n", op.Name, op.Background, op.WeakDeps) + } + } + } + return +} + +// LogIfError will log if there is an error with the given context as message +// Context can be empty +func (s *State) LogIfError(e error, msgContext string) { + if e != nil { + s.Logger.Err(e).Msg(msgContext) + } +} + +// LogIfErrorAndReturn will log if there is an error with the given context as message +// Context can be empty +// Will also return the error +func (s *State) LogIfErrorAndReturn(e error, msgContext string) error { + if e != nil { + s.Logger.Err(e).Msg(msgContext) + } + return e +} + +// LogIfErrorAndPanic will log if there is an error with the given context as message +// Context can be empty +// Will also panic +func (s *State) LogIfErrorAndPanic(e error, msgContext string) { + if e != nil { + s.Logger.Err(e).Msg(msgContext) + s.Logger.Fatal().Msg(e.Error()) + } +} diff --git a/pkg/mount/mount_suite_test.go b/pkg/mount/state_suite_test.go similarity index 100% rename from pkg/mount/mount_suite_test.go rename to pkg/mount/state_suite_test.go diff --git a/pkg/mount/state_test.go b/pkg/mount/state_test.go new file mode 100644 index 0000000..ffabcbd --- /dev/null +++ b/pkg/mount/state_test.go @@ -0,0 +1,137 @@ +package mount_test + +import ( + "context" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "os" + "time" + + "github.com/kairos-io/immucore/pkg/mount" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spectrocloud-labs/herd" +) + +var _ = Describe("mounting immutable setup", func() { + var g *herd.Graph + + BeforeEach(func() { + g = herd.DAG(herd.EnableInit) + Expect(g).ToNot(BeNil()) + }) + + Context("simple invocation", func() { + It("generates normal dag", func() { + s := &mount.State{ + Logger: log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).With().Logger(), + Rootdir: "/", + TargetImage: "/cOS/myimage.img", + TargetLabel: "COS_LABEL", + MountRoot: true, + } + + err := s.RegisterNormalBoot(g) + Expect(err).ToNot(HaveOccurred()) + + dag := g.Analyze() + + checkDag(dag, s.WriteDAG(g)) + + }) + It("generates normal dag with extra dirs", func() { + s := &mount.State{Rootdir: "/", MountRoot: true, + OverlayDirs: []string{"/etc"}, + BindMounts: []string{"/etc/kubernetes"}, + CustomMounts: map[string]string{"COS_PERSISTENT": "/usr/local"}} + + err := s.RegisterNormalBoot(g) + Expect(err).ToNot(HaveOccurred()) + + dag := g.Analyze() + + checkDag(dag, s.WriteDAG(g)) + }) + It("generates livecd dag", func() { + s := &mount.State{} + err := s.RegisterLiveMedia(g) + Expect(err).ToNot(HaveOccurred()) + dag := g.Analyze() + checkLiveCDDag(dag, s.WriteDAG(g)) + + }) + + It("Mountop timeouts", func() { + s := &mount.State{} + f := s.MountOP("/dev/doesntexist", "/tmp/jojobizarreadventure", "", []string{}, 500*time.Millisecond) + err := f(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exhausted")) + }) + }) +}) + +func checkLiveCDDag(dag [][]herd.GraphEntry, actualDag string) { + Expect(len(dag)).To(Equal(2), actualDag) + Expect(len(dag[0])).To(Equal(1), actualDag) + Expect(len(dag[1])).To(Equal(2), actualDag) + + Expect(dag[0][0].Name).To(Equal("init")) + Expect(dag[1][0].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + ), actualDag) + Expect(dag[1][1].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + ), actualDag) + +} +func checkDag(dag [][]herd.GraphEntry, actualDag string) { + Expect(len(dag)).To(Equal(10), actualDag) + Expect(len(dag[0])).To(Equal(1), actualDag) + Expect(len(dag[1])).To(Equal(4), actualDag) + Expect(len(dag[2])).To(Equal(1), actualDag) + Expect(len(dag[3])).To(Equal(1), actualDag) + Expect(len(dag[4])).To(Equal(1), actualDag) + Expect(len(dag[5])).To(Equal(1), actualDag) + Expect(len(dag[6])).To(Equal(1), actualDag) + Expect(len(dag[7])).To(Equal(2), actualDag) + Expect(len(dag[8])).To(Equal(1), actualDag) + Expect(len(dag[9])).To(Equal(1), actualDag) + + Expect(dag[0][0].Name).To(Equal("init")) + Expect(dag[1][0].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + Equal("mount-base-overlay"), + Equal("mount-state"), + ), actualDag) + Expect(dag[1][1].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + Equal("mount-base-overlay"), + Equal("mount-state"), + ), actualDag) + Expect(dag[1][2].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + Equal("mount-base-overlay"), + Equal("mount-state"), + ), actualDag) + Expect(dag[1][3].Name).To(Or( + Equal("mount-tmpfs"), + Equal("create-sentinel"), + Equal("mount-base-overlay"), + Equal("mount-state"), + ), actualDag) + Expect(dag[2][0].Name).To(Equal("discover-state"), actualDag) + Expect(dag[3][0].Name).To(Equal("mount-root"), actualDag) + Expect(dag[4][0].Name).To(Equal("mount-oem"), actualDag) + Expect(dag[5][0].Name).To(Equal("rootfs-hook"), actualDag) + Expect(dag[6][0].Name).To(Equal("load-config"), actualDag) + Expect(dag[7][0].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), actualDag) + Expect(dag[7][1].Name).To(Or(Equal("overlay-mount"), Equal("custom-mount")), actualDag) + Expect(dag[8][0].Name).To(Equal("mount-bind"), actualDag) + Expect(dag[9][0].Name).To(Equal("write-fstab"), actualDag) +}