Rework workflow (#77)

This makes immucore run more in parallel rather than block everything
else. We just tell sysroot.mount that eventually it will be mounted and
to wait for a bit.

This allows us to be more flexible where to run and run in parallel in
cases like cdrom in which we may do things but we need the sysroot to be
mounted already but not from us.

Also adds the initramfs stage directly in immucore and merges all the dracut config into one

    Dont create sysroot, just add a timeout override so it waits for us
    Dont block on the service, just make sure to finish before
    initrd.target
    Fix mounts from cmdline
    More proper log
    Store logs under the /run/immucore dir
    Store rootfs and initramfs logs separated
    Do not log the full stages in INFO level
    Run initramfs stage in immucore directly on boot and cd/netboot
    Drop systemd requirement from dracut module

Signed-off-by: Itxaka itxaka.garcia@spectrocloud.com

Earthfile

@@ -73,10 +73,6 @@ dracut-artifacts:
     FROM $BASE_IMAGE
     WORKDIR /build
     COPY --dir dracut/28immucore .
-    COPY dracut/02-kairos-setup-initramfs.conf .
     COPY dracut/10-immucore.conf .
-    COPY dracut/50-kairos-initrd.conf .
     SAVE ARTIFACT 28immucore 28immucore
-    SAVE ARTIFACT 02-kairos-setup-initramfs.conf 02-kairos-setup-initramfs.conf
-    SAVE ARTIFACT 10-immucore.conf 10-immucore.conf
-    SAVE ARTIFACT 50-kairos-initrd.conf 50-kairos-initrd.conf
+    SAVE ARTIFACT 10-immucore.conf 10-immucore.conf

README.md

@@ -237,6 +237,8 @@ There is also the `weak` value which indicates that this step has weak dependenc
  - `custom-mount`: This mounts the paths set in the config (`VOLUMES`) or in cmdline `rd.cos.mount=` in the given path (`LABEL=COS_PERSISTENT:/usr/local`)
  - `mount-bind`: This mounts the paths set in the config (`PERSISTENT_STATE_PATHS` and `CUSTOM_BIND_MOUNTS`) as bind mounts under the `PERSISTENT_STATE_TARGET` which defaults to `/usr/local/.state`
  - `write-fstab`: Writes the final fstab with all the mounts into `/sysroot/fstab`
+ - `initramfs-hook`: Runs the cloud config stage `initramfs`. Note that this is run under a chroot into what will be the final system (/sysroot).
+ - `wait-for-sysroot`: Waits for the /sysroot and /sysroot/system dirs to be available, which means that they are mounted. Useful when booting from CD/Netboot as immucore doesn't mount the /sysroot in those cases, but we want to run the initramfs stage once the system is ready.
 
 ### UKI mode (Experimental)
 

dracut/02-kairos-setup-initramfs.conf

@@ -1,2 +0,0 @@
-install_items+=" /etc/hosts "
-add_dracutmodules+=" network "

dracut/10-immucore.conf

@@ -1 +1,8 @@
-add_dracutmodules+=" immucore "
+hostonly_cmdline="no"
+hostonly="no"
+compress="xz"
+i18n_install_all="yes"
+show_modules="yes"
+install_items+=" /etc/hosts "
+omit_dracutmodules+=" multipath "
+add_dracutmodules+=" livenet dmsquash-live immucore network "

dracut/28immucore/generator.sh

@@ -8,25 +8,19 @@ GENERATOR_DIR="$2"
 [ -z "$GENERATOR_DIR" ] && exit 1
 [ -d "$GENERATOR_DIR" ] || mkdir "$GENERATOR_DIR"
 
-
-## GENERATE SYSROOT
-cos_img=$(getarg cos-img/filename=)
-[ -z "${cos_img}" ] && exit 0
-
-# This is necessary because otherwise systemd-fstab-generator will se the cmdline with the root=LABEL=X stanza and
-# say, hey this is the ROOT where we need to boot! so it auto creates a sysroot.mount with the content of the value
-# passed in the cmdline. But because we usually pass the label of the img (COS_ACTIVE) it will create the wrong mount
-# service and be stuck in there forever.
-# by generating it ourselves we get the sysroot.mount into the generators.early dir, which tells systemd to not generate it
-# as it already exists and the rest is history
+# Add a timeout to the sysroot so it waits a bit for immucore to mount it properly
+mkdir -p "$GENERATOR_DIR"/sysroot.mount.d
 {
-    echo "[Unit]"
-    echo "Before=initrd-root-fs.target"
-    echo "DefaultDependencies=no"
     echo "[Mount]"
-    echo "Where=/sysroot"
-    echo "What=/run/initramfs/cos-state/${cos_img#/}"
-    echo "Options=ro,suid,dev,exec,auto,nouser,async"
-} > "$GENERATOR_DIR"/sysroot.mount
+    echo "TimeoutSec=300"
+} > "$GENERATOR_DIR"/sysroot.mount.d/timeout.conf
 
-## END GENERATE SYSROOT
+# Make sure initrd-root-fs.target depends on sysroot.mount
+# This seems to affect mainly ubuntu-22 where initrd-usr-fs depends on sysroot, but it has a broken link to it as sysroot.mount
+# is generated under the generator.early dir but the link points to the generator dir.
+# So it makes everything else a bit broken if you insert deps in the middle.
+# By default other distros seem to do this as it shows on the map page https://man7.org/linux/man-pages/man7/dracut.bootup.7.html
+if ! [ -L "$GENERATOR_DIR"/initrd-root-fs.target.wants/sysroot.mount ]; then
+  [ -d "$GENERATOR_DIR"/initrd-root-fs.target.wants ] || mkdir -p "$GENERATOR_DIR"/initrd-root-fs.target.wants
+  ln -s ../sysroot.mount "$GENERATOR_DIR"/initrd-root-fs.target.wants/sysroot.mount
+fi

dracut/28immucore/immucore.service

@@ -3,7 +3,7 @@ Description=immucore
 DefaultDependencies=no
 After=systemd-udev-settle.service
 Requires=systemd-udev-settle.service
-Before=dracut-initqueue.service sysroot.mount
+Before=initrd-fs.target
 Conflicts=initrd-switch-root.target
 
 [Service]

dracut/28immucore/kairos-setup-initramfs.service

@@ -1,16 +0,0 @@
-[Unit]
-Description=kairos system initramfs setup before switch root
-DefaultDependencies=no
-After=initrd-fs.target
-Requires=initrd-fs.target
-Before=initrd.target
-
-[Service]
-RootDirectory=/sysroot
-BindPaths=/proc /sys /dev /run /tmp
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/bin/elemental run-stage initramfs
-
-[Install]
-RequiredBy=initrd.target

dracut/28immucore/module-setup.sh

@@ -2,14 +2,12 @@
 
 # called by dracut
 check() {
-    require_binaries "$systemdutildir"/systemd || return 1
-    return 255
+    return 0
 }
 
 # called by dracut 
 depends() {
-    echo systemd rootfs-block dm fs-lib 
-    #tpm2-tss
+    echo rootfs-block dm fs-lib
     return 0
 }
 
@@ -31,13 +29,8 @@ install() {
     # missing mkfs.xfs xfs_growfs in image?
     inst_script "${moddir}/generator.sh" "${systemdutildir}/system-generators/immucore-generator"
     inst_simple "${moddir}/immucore.service" "${systemdsystemunitdir}/immucore.service"
-    mkdir -p "${initdir}/${systemdsystemunitdir}/initrd-fs.target.requires"
-    ln_r "../immucore.service" "${systemdsystemunitdir}/initrd-fs.target.requires/immucore.service"
-
-    # Until this is done on immucore, we need to ship it as part of the dracut module
-    inst_simple "${moddir}/kairos-setup-initramfs.service" "${systemdsystemunitdir}/kairos-setup-initramfs.service"
     mkdir -p "${initdir}/${systemdsystemunitdir}/initrd.target.requires"
-    ln_r "../kairos-setup-initramfs.service" "${systemdsystemunitdir}/initrd.target.requires/kairos-setup-initramfs.service"
+    ln_r "../immucore.service" "${systemdsystemunitdir}/initrd.target.requires/immucore.service"
 
     dracut_need_initqueue
 }

dracut/50-kairos-initrd.conf

@@ -1,5 +0,0 @@
-hostonly_cmdline="no"
-hostonly="no"
-compress="xz"
-omit_dracutmodules+=" multipath "
-add_dracutmodules+=" livenet dmsquash-live "

internal/constants/constants.go

@@ -27,5 +27,7 @@ const (
 	OpUkiInit             = "uki-init"
 	OpSentinel            = "create-sentinel"
 	OpUkiUdev             = "uki-udev"
+	OpWaitForSysroot      = "wait-for-sysroot"
 	PersistentStateTarget = "/usr/local/.state"
+	LogDir                = "/run/immucore"
 )

internal/utils/chroot.go

@@ -0,0 +1,191 @@
+/*
+Copyright © 2022 SUSE LLC
+Copyright © 2023 Kairos authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+)
+
+// Chroot represents the struct that will allow us to run commands inside a given chroot.
+type Chroot struct {
+	path          string
+	defaultMounts []string
+	activeMounts  []string
+}
+
+func NewChroot(path string) *Chroot {
+	return &Chroot{
+		path:          path,
+		defaultMounts: []string{"/dev", "/proc", "/sys", "/run", "/tmp"},
+		activeMounts:  []string{},
+	}
+}
+
+// ChrootedCallback runs the given callback in a chroot environment.
+func ChrootedCallback(path string, bindMounts map[string]string, callback func() error) error {
+	chroot := NewChroot(path)
+	return chroot.RunCallback(callback)
+}
+
+// Prepare will mount the defaultMounts as bind mounts, to be ready when we run chroot.
+func (c *Chroot) Prepare() error {
+	var err error
+
+	if len(c.activeMounts) > 0 {
+		return errors.New("there are already active mountpoints for this instance")
+	}
+
+	defer func() {
+		if err != nil {
+			c.Close()
+		}
+	}()
+
+	for _, mnt := range c.defaultMounts {
+		mountPoint := filepath.Join(c.path, mnt)
+		err = CreateIfNotExists(mountPoint)
+		if err != nil {
+			Log.Err(err).Str("what", mountPoint).Msg("Creating dir")
+			return err
+		}
+		err = syscall.Mount(mnt, mountPoint, "bind", syscall.MS_BIND|syscall.MS_REC, "")
+		if err != nil {
+			Log.Err(err).Str("where", mountPoint).Str("what", mnt).Msg("Mounting chroot bind")
+			return err
+		}
+		c.activeMounts = append(c.activeMounts, mountPoint)
+	}
+
+	return nil
+}
+
+// Close will unmount all active mounts created in Prepare on reverse order.
+func (c *Chroot) Close() error {
+	failures := []string{}
+	for len(c.activeMounts) > 0 {
+		curr := c.activeMounts[len(c.activeMounts)-1]
+		Log.Debug().Str("what", curr).Msg("Unmounting from chroot")
+		c.activeMounts = c.activeMounts[:len(c.activeMounts)-1]
+		err := syscall.Unmount(curr, 0)
+		if err != nil {
+			Log.Err(err).Str("what", curr).Msg("Error unmounting")
+			failures = append(failures, curr)
+		}
+	}
+	if len(failures) > 0 {
+		c.activeMounts = failures
+		return fmt.Errorf("failed closing chroot environment. Unmount failures: %v", failures)
+	}
+	return nil
+}
+
+// RunCallback runs the given callback in a chroot environment.
+func (c *Chroot) RunCallback(callback func() error) (err error) {
+	var currentPath string
+	var oldRootF *os.File
+
+	// Store current path
+	currentPath, err = os.Getwd()
+	if err != nil {
+		Log.Err(err).Msg("Failed to get current path")
+		return err
+	}
+	defer func() {
+		tmpErr := os.Chdir(currentPath)
+		if err == nil && tmpErr != nil {
+			err = tmpErr
+		}
+	}()
+
+	// Store current root
+	oldRootF, err = os.Open("/")
+	if err != nil {
+		Log.Err(err).Msg("Can't open current root")
+		return err
+	}
+	defer oldRootF.Close()
+
+	if len(c.activeMounts) == 0 {
+		err = c.Prepare()
+		if err != nil {
+			Log.Err(err).Msg("Can't mount default mounts")
+			return err
+		}
+		defer func() {
+			tmpErr := c.Close()
+			if err == nil {
+				err = tmpErr
+			}
+		}()
+	}
+	// Change to new dir before running chroot!
+	err = syscall.Chdir(c.path)
+	if err != nil {
+		Log.Err(err).Str("path", c.path).Msg("Can't chdir")
+		return err
+	}
+
+	err = syscall.Chroot(c.path)
+	if err != nil {
+		Log.Err(err).Str("path", c.path).Msg("Can't chroot")
+		return err
+	}
+
+	// Restore to old root
+	defer func() {
+		tmpErr := oldRootF.Chdir()
+		if tmpErr != nil {
+			Log.Err(tmpErr).Str("path", oldRootF.Name()).Msg("Can't change to old root dir")
+			if err == nil {
+				err = tmpErr
+			}
+		} else {
+			tmpErr = syscall.Chroot(".")
+			if tmpErr != nil {
+				Log.Err(tmpErr).Str("path", oldRootF.Name()).Msg("Can't chroot back to old root")
+				if err == nil {
+					err = tmpErr
+				}
+			}
+		}
+	}()
+
+	return callback()
+}
+
+// Run executes a command inside a chroot.
+func (c *Chroot) Run(command string) (string, error) {
+	var err error
+	var out []byte
+	callback := func() error {
+		cmd := exec.Command("/bin/sh", "-c", command)
+		cmd.Env = os.Environ()
+		out, err = cmd.CombinedOutput()
+		return err
+	}
+	err = c.RunCallback(callback)
+	if err != nil {
+		Log.Err(err).Str("cmd", command).Msg("Cant run command on chroot")
+	}
+	return string(out), err
+}

internal/utils/common.go

@@ -182,7 +182,6 @@ func CommandWithPath(c string) (string, error) {
 			pathAppend = fmt.Sprintf("%s:%s", pathAppend, splitted[1])
 		}
 	}
-	Log.Debug().Str("content", pathAppend).Msg("PATH")
 	cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", pathAppend))
 	o, err := cmd.CombinedOutput()
 	return string(o), err

internal/utils/log.go

@@ -3,7 +3,9 @@ package utils
 import (
 	"io"
 	"os"
+	"path/filepath"
 
+	"github.com/kairos-io/immucore/internal/constants"
 	"github.com/rs/zerolog"
 )
 
@@ -16,7 +18,8 @@ func CloseLogFiles() {
 
 func SetLogger() {
 	var loggers []io.Writer
-	logFile, err := os.Create("/run/immucore.log")
+	_ = os.MkdirAll(constants.LogDir, os.ModeDir|os.ModePerm)
+	logFile, err := os.Create(filepath.Join(constants.LogDir, "immucore.log"))
 	if err == nil {
 		loggers = append(loggers, zerolog.ConsoleWriter{Out: logFile})
 	}

internal/utils/mounts.go

@@ -169,9 +169,8 @@ func Fsck(device string) error {
 	cmd := strings.Join(args, " ")
 	Log.Debug().Str("cmd", cmd).Msg("fsck command")
 	out, e := CommandWithPath(cmd)
-	Log.Debug().Str("output", out).Msg("fsck output")
 	if e != nil {
-		Log.Warn().Str("error", e.Error()).Str("what", device).Msg("fsck")
+		Log.Debug().Err(e).Str("out", out).Str("what", device).Msg("fsck")
 	}
 	return e
 }

pkg/mount/dag_live_media.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	cnst "github.com/kairos-io/immucore/internal/constants"
 	"github.com/spectrocloud-labs/herd"
 )
 
@@ -9,5 +10,12 @@ import (
 func (s *State) RegisterLiveMedia(g *herd.Graph) error {
 	// Maybe LogIfErrorAndPanic ? If no sentinel, a lot of config files are not going to run
 	err := s.LogIfErrorAndReturn(s.WriteSentinelDagStep(g), "write sentinel")
+
+	// Waits for sysroot to be there, just in case
+	s.LogIfError(s.WaitForSysrootDagStep(g), "Waiting for sysroot")
+	// Run rootfs
+	s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel, cnst.OpWaitForSysroot), "rootfs stage")
+	// Run initramfs inside the /sysroot chroot!
+	s.LogIfError(s.InitramfsStageDagStep(g, herd.WithDeps(cnst.OpSentinel, cnst.OpWaitForSysroot, cnst.OpRootfsHook), herd.WithWeakDeps()), "initramfs stage")
 	return err
 }

pkg/mount/dag_normal_boot.go

@@ -51,5 +51,10 @@ func (s *State) RegisterNormalBoot(g *herd.Graph) error {
 
 	// Write fstab file
 	s.LogIfError(s.WriteFstabDagStep(g), "write fstab")
+	// do it after fstab is created
+	s.LogIfError(s.InitramfsStageDagStep(g,
+		herd.WithDeps(cnst.OpMountRoot, cnst.OpDiscoverState, cnst.OpLoadConfig, cnst.OpWriteFstab),
+		herd.WithWeakDeps(cnst.OpMountBaseOverlay, cnst.OpMountOEM, cnst.OpMountBind, cnst.OpMountBind, cnst.OpCustomMounts, cnst.OpOverlayMount),
+	), "initramfs stage")
 	return err
 }

pkg/mount/dag_steps.go

@@ -107,8 +107,8 @@ func (s *State) RootfsStageDagStep(g *herd.Graph, deps ...string) error {
 }
 
 // InitramfsStageDagStep will add the rootfs stage.
-func (s *State) InitramfsStageDagStep(g *herd.Graph, deps ...string) error {
-	return g.Add(cnst.OpInitramfsHook, herd.WithDeps(deps...), herd.WeakDeps, herd.WithCallback(s.RunStageOp("initramfs")))
+func (s *State) InitramfsStageDagStep(g *herd.Graph, deps herd.OpOption, weakDeps herd.OpOption) error {
+	return g.Add(cnst.OpInitramfsHook, deps, weakDeps, herd.WithCallback(s.RunStageOp("initramfs")))
 }
 
 // LoadEnvLayoutDagStep will add the stage to load from cos-layout.env and fill the proper CustomMounts, OverlayDirs and BindMounts.
@@ -163,11 +163,8 @@ func (s *State) LoadEnvLayoutDagStep(g *herd.Graph, deps ...string) error {
 			// Parse custom mounts also from cmdline (rd.cos.mount=)
 			// Parse custom mounts also from cmdline (rd.immucore.mount=)
 			// Parse custom mounts also from env file (VOLUMES)
-			var mounts []string
-			mounts = internalUtils.CleanupSlice(internalUtils.ReadCMDLineArg("rd.cos.mount="))
-			mounts = append(mounts, internalUtils.CleanupSlice(internalUtils.ReadCMDLineArg("rd.immucore.mount="))...)
-			mounts = append(mounts, env["VOLUMES"])
-			for _, v := range mounts {
+
+			for _, v := range append(append(internalUtils.ReadCMDLineArg("rd.cos.mount="), internalUtils.ReadCMDLineArg("rd.immucore.mount=")...), strings.Split(env["VOLUMES"], " ")...) {
 				addLine(internalUtils.ParseMount(v))
 			}
 
@@ -257,6 +254,7 @@ func (s *State) MountCustomMountsDagStep(g *herd.Graph) error {
 		herd.WithDeps(cnst.OpLoadConfig),
 		herd.WithCallback(func(ctx context.Context) error {
 			var err *multierror.Error
+			internalUtils.Log.Debug().Interface("mounts", s.CustomMounts).Msg("Mounting custom mounts")
 
 			for what, where := range s.CustomMounts {
 				// TODO: scan for the custom mount disk to know the underlying fs and set it proper
@@ -471,3 +469,37 @@ func (s *State) LoadKernelModules(g *herd.Graph) error {
 		}),
 	)
 }
+
+// WaitForSysrootDagStep waits for the s.Rootdir and s.Rootdir/system paths to be there
+// Useful for livecd/netboot as we want to run steps after s.Rootdir is ready but we don't mount it ourselves.
+func (s *State) WaitForSysrootDagStep(g *herd.Graph) error {
+	return g.Add(cnst.OpWaitForSysroot,
+		herd.WithCallback(func(ctx context.Context) error {
+			cc := time.After(60 * time.Second)
+			for {
+				select {
+				default:
+					time.Sleep(2 * time.Second)
+					_, err := os.Stat(s.Rootdir)
+					if err != nil {
+						internalUtils.Log.Debug().Str("what", s.Rootdir).Msg("Checking path existence")
+						continue
+					}
+					_, err = os.Stat(filepath.Join(s.Rootdir, "system"))
+					if err != nil {
+						internalUtils.Log.Debug().Str("what", filepath.Join(s.Rootdir, "system")).Msg("Checking path existence")
+						continue
+					}
+					return nil
+				case <-ctx.Done():
+					e := fmt.Errorf("context canceled")
+					internalUtils.Log.Err(e).Str("what", s.Rootdir).Msg("filepath check canceled")
+					return e
+				case <-cc:
+					e := fmt.Errorf("timeout exhausted")
+					internalUtils.Log.Err(e).Str("what", s.Rootdir).Msg("filepath check timeout")
+					return e
+				}
+			}
+		}))
+}

pkg/mount/dag_uki_boot.go

@@ -39,7 +39,7 @@ func (s *State) RegisterUKI(g *herd.Graph) error {
 	s.LogIfError(s.MountCustomBindsDagStep(g), "custom binds mount")
 
 	// run initramfs stage
-	s.LogIfError(s.InitramfsStageDagStep(g, cnst.OpMountBind), "uki initramfs")
+	s.LogIfError(s.InitramfsStageDagStep(g, herd.WithDeps(cnst.OpMountBind), herd.WithWeakDeps()), "uki initramfs")
 
 	s.LogIfError(g.Add(cnst.OpWriteFstab,
 		herd.WithDeps(cnst.OpLoadConfig, cnst.OpCustomMounts, cnst.OpMountBind, cnst.OpOverlayMount),

pkg/mount/fs.go

@@ -80,7 +80,7 @@ func mountBind(mountpoint, root, stateTarget string) mountOperation {
 			"bind",
 		},
 	}
-	internalUtils.Log.Debug().Str("mountpoint", mountpoint).Str("root", root).Str("bindMountPath", bindMountPath).Msg("BIND")
+	internalUtils.Log.Debug().Str("where", rootMount).Str("what", stateDir).Msg("Bind mount")
 	tmpFstab := internalUtils.MountToFstab(tmpMount)
 	tmpFstab.File = internalUtils.CleanSysrootForFstab(fmt.Sprintf("/%s", mountpoint))
 	tmpFstab.Spec = internalUtils.CleanSysrootForFstab(tmpFstab.Spec)

pkg/mount/operation.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kairos-io/immucore/internal/constants"
 	internalUtils "github.com/kairos-io/immucore/internal/utils"
 	"github.com/moby/sys/mountinfo"
+	"github.com/rs/zerolog"
 )
 
 type mountOperation struct {
@@ -16,23 +17,30 @@ type mountOperation struct {
 }
 
 func (m mountOperation) run() error {
-	internalUtils.Log.With().Str("what", m.MountOption.Source).Str("where", m.Target).Str("type", m.MountOption.Type).Strs("options", m.MountOption.Options).Logger()
+	// Add context to sublogger
+	l := internalUtils.Log.With().Str("what", m.MountOption.Source).Str("where", m.Target).Str("type", m.MountOption.Type).Strs("options", m.MountOption.Options).Logger().Level(zerolog.InfoLevel)
+	// Not sure why this defaults to debuglevel when creating a sublogger, so make sure we set it properly
+	debug := len(internalUtils.ReadCMDLineArg("rd.immucore.debug")) > 0
+	if debug {
+		l.Level(zerolog.DebugLevel)
+	}
+
 	if m.PrepareCallback != nil {
 		if err := m.PrepareCallback(); err != nil {
-			internalUtils.Log.Err(err).Msg("executing mount callback")
+			l.Err(err).Msg("executing mount callback")
 			return err
 		}
 	}
 	//TODO: not only check if mounted but also if the type,options and source are the same?
 	mounted, err := mountinfo.Mounted(m.Target)
 	if err != nil {
-		internalUtils.Log.Err(err).Msg("checking mount status")
+		l.Err(err).Msg("checking mount status")
 		return err
 	}
 	if mounted {
-		internalUtils.Log.Debug().Msg("Already mounted")
+		l.Debug().Msg("Already mounted")
 		return constants.ErrAlreadyMounted
 	}
-	internalUtils.Log.Debug().Msg("mount ready")
+	l.Debug().Msg("mount ready")
 	return mount.All([]mount.Mount{m.MountOption}, m.Target)
 }

pkg/mount/state.go

@@ -41,6 +41,12 @@ func (s *State) path(p ...string) string {
 
 func (s *State) WriteFstab(fstabFile string) func(context.Context) error {
 	return func(ctx context.Context) error {
+		// Create the file first, override if something is there, we don't care, we are on initramfs
+		f, err := os.Create(fstabFile)
+		if err != nil {
+			return err
+		}
+		f.Close()
 		for _, fst := range s.fstabs {
 			select {
 			case <-ctx.Done():
@@ -65,35 +71,63 @@ func (s *State) WriteFstab(fstabFile string) func(context.Context) error {
 // If its uki we don't symlink as we already have everything in the sysroot.
 func (s *State) RunStageOp(stage string) func(context.Context) error {
 	return func(ctx context.Context) error {
-		if stage == "rootfs" && !internalUtils.IsUKI() {
-			if _, err := os.Stat("/system"); os.IsNotExist(err) {
-				err = os.Symlink("/sysroot/system", "/system")
-				if err != nil {
-					internalUtils.Log.Err(err).Msg("creating symlink")
-				}
-			}
-			if _, err := os.Stat("/oem"); os.IsNotExist(err) {
-				err = os.Symlink("/sysroot/oem", "/oem")
-				if err != nil {
-					internalUtils.Log.Err(err).Msg("creating symlink")
-				}
-			}
-		}
-
-		cmd := fmt.Sprintf("/usr/bin/elemental run-stage %s", stage)
+		cmd := fmt.Sprintf("elemental run-stage %s", stage)
 		// If we set the level to debug, also call elemental with debug
 		if internalUtils.Log.GetLevel() == zerolog.DebugLevel {
 			cmd = fmt.Sprintf("%s --debug", cmd)
 		}
-		output, err := utils.SH(cmd)
-		internalUtils.Log.Debug().Msg(output)
-		return err
+
+		switch stage {
+		case "rootfs":
+			if !internalUtils.IsUKI() {
+				if _, err := os.Stat("/system"); os.IsNotExist(err) {
+					err = os.Symlink("/sysroot/system", "/system")
+					if err != nil {
+						internalUtils.Log.Err(err).Msg("creating symlink")
+					}
+				}
+				if _, err := os.Stat("/oem"); os.IsNotExist(err) {
+					err = os.Symlink("/sysroot/oem", "/oem")
+					if err != nil {
+						internalUtils.Log.Err(err).Msg("creating symlink")
+					}
+				}
+			}
+			output, err := utils.SH(cmd)
+			internalUtils.Log.Info().Msg("Running rootfs stage")
+			internalUtils.Log.Info().Msg(output)
+			f, ferr := os.Create(filepath.Join(constants.LogDir, "rootfs_stage.log"))
+			if ferr == nil {
+				_, _ = f.WriteString(output)
+				_ = f.Close()
+			}
+			return err
+		case "initramfs":
+			// Not sure if it will work under UKI where the s.Rootdir is the current root already
+			chroot := internalUtils.NewChroot(s.Rootdir)
+			output, err := chroot.Run(cmd)
+			internalUtils.Log.Info().Msg("Running initramfs stage")
+			internalUtils.Log.Info().Msg(output)
+			f, ferr := os.Create(filepath.Join(constants.LogDir, "initramfs_stage.log"))
+			if ferr == nil {
+				_, _ = f.WriteString(output)
+				_ = f.Close()
+			}
+			return err
+		default:
+			return errors.New("no stage that we know off")
+		}
 	}
 }
 
 // MountOP creates and executes a mount operation.
 func (s *State) MountOP(what, where, t string, options []string, timeout time.Duration) func(context.Context) error {
-	internalUtils.Log.With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger()
+	l := internalUtils.Log.With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger().Level(zerolog.InfoLevel)
+	// Not sure why this defaults to debuglevel when creating a sublogger, so make sure we set it properly
+	debug := len(internalUtils.ReadCMDLineArg("rd.immucore.debug")) > 0
+	if debug {
+		l.Level(zerolog.DebugLevel)
+	}
 
 	return func(c context.Context) error {
 		cc := time.After(timeout)
@@ -102,7 +136,7 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du
 			default:
 				err := internalUtils.CreateIfNotExists(where)
 				if err != nil {
-					internalUtils.Log.Err(err).Msg("Creating dir")
+					l.Err(err).Msg("Creating dir")
 					continue
 				}
 				time.Sleep(1 * time.Second)
@@ -131,18 +165,18 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du
 
 				// only continue the loop if it's an error and not an already mounted error
 				if err != nil && !errors.Is(err, constants.ErrAlreadyMounted) {
-					internalUtils.Log.Err(err).Send()
+					l.Err(err).Send()
 					continue
 				}
-				internalUtils.Log.Debug().Msg("mount done")
+				l.Info().Msg("mount done")
 				return nil
 			case <-c.Done():
 				e := fmt.Errorf("context canceled")
-				internalUtils.Log.Err(e).Msg("mount canceled")
+				l.Err(e).Msg("mount canceled")
 				return e
 			case <-cc:
 				e := fmt.Errorf("timeout exhausted")
-				internalUtils.Log.Err(e).Msg("Mount timeout")
+				l.Err(e).Msg("Mount timeout")
 				return e
 			}
 		}
@@ -155,9 +189,9 @@ func (s *State) WriteDAG(g *herd.Graph) (out string) {
 		out += fmt.Sprintf("%d.\n", i+1)
 		for _, op := range layer {
 			if op.Error != nil {
-				out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps)
+				out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t) (run: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps, op.Executed)
 			} else {
-				out += fmt.Sprintf(" <%s> (background: %t) (weak: %t)\n", op.Name, op.Background, op.WeakDeps)
+				out += fmt.Sprintf(" <%s> (background: %t) (weak: %t) (run: %t)\n", op.Name, op.Background, op.WeakDeps, op.Executed)
 			}
 		}
 	}

pkg/mount/state_test.go

@@ -2,6 +2,7 @@ package mount_test
 
 import (
 	"context"
+	cnst "github.com/kairos-io/immucore/internal/constants"
 	"time"
 
 	"github.com/kairos-io/immucore/pkg/mount"
@@ -67,16 +68,21 @@ var _ = Describe("mounting immutable setup", func() {
 })
 
 func checkLiveCDDag(dag [][]herd.GraphEntry, actualDag string) {
-	Expect(len(dag)).To(Equal(2), actualDag)
+	Expect(len(dag)).To(Equal(4), actualDag)
 	Expect(len(dag[0])).To(Equal(1), actualDag)
-	Expect(len(dag[1])).To(Equal(1), actualDag)
+	Expect(len(dag[1])).To(Equal(2), actualDag)
+	Expect(len(dag[2])).To(Equal(1), actualDag)
+	Expect(len(dag[3])).To(Equal(1), actualDag)
 
 	Expect(dag[0][0].Name).To(Equal("init"))
-	Expect(dag[1][0].Name).To(Equal("create-sentinel"))
+	Expect(dag[1][0].Name).To(Or(Equal(cnst.OpSentinel), Equal(cnst.OpWaitForSysroot)), actualDag)
+	Expect(dag[1][1].Name).To(Or(Equal(cnst.OpSentinel), Equal(cnst.OpWaitForSysroot)), actualDag)
+	Expect(dag[2][0].Name).To(Equal(cnst.OpRootfsHook), actualDag)
+	Expect(dag[3][0].Name).To(Equal(cnst.OpInitramfsHook), actualDag)
 
 }
 func checkDag(dag [][]herd.GraphEntry, actualDag string) {
-	Expect(len(dag)).To(Equal(10), actualDag)
+	Expect(len(dag)).To(Equal(11), actualDag)
 	Expect(len(dag[0])).To(Equal(1), actualDag)
 	Expect(len(dag[1])).To(Equal(3), actualDag)
 	Expect(len(dag[2])).To(Equal(1), actualDag)
@@ -87,31 +93,33 @@ func checkDag(dag [][]herd.GraphEntry, actualDag string) {
 	Expect(len(dag[7])).To(Equal(2), actualDag)
 	Expect(len(dag[8])).To(Equal(2), actualDag)
 	Expect(len(dag[9])).To(Equal(1), actualDag)
+	Expect(len(dag[10])).To(Equal(1), actualDag)
 
 	Expect(dag[0][0].Name).To(Equal("init"))
 	Expect(dag[1][0].Name).To(Or(
-		Equal("mount-tmpfs"),
-		Equal("create-sentinel"),
-		Equal("mount-state"),
+		Equal(cnst.OpMountTmpfs),
+		Equal(cnst.OpSentinel),
+		Equal(cnst.OpMountState),
 	), actualDag)
 	Expect(dag[1][1].Name).To(Or(
-		Equal("mount-tmpfs"),
-		Equal("create-sentinel"),
-		Equal("mount-state"),
+		Equal(cnst.OpMountTmpfs),
+		Equal(cnst.OpSentinel),
+		Equal(cnst.OpMountState),
 	), actualDag)
 	Expect(dag[1][2].Name).To(Or(
-		Equal("mount-tmpfs"),
-		Equal("create-sentinel"),
-		Equal("mount-state"),
+		Equal(cnst.OpMountTmpfs),
+		Equal(cnst.OpSentinel),
+		Equal(cnst.OpMountState),
 	), actualDag)
-	Expect(dag[2][0].Name).To(Equal("discover-state"), actualDag)
-	Expect(dag[3][0].Name).To(Equal("mount-root"), actualDag)
-	Expect(dag[4][0].Name).To(Equal("mount-oem"), actualDag)
-	Expect(dag[5][0].Name).To(Equal("rootfs-hook"), actualDag)
-	Expect(dag[6][0].Name).To(Equal("load-config"), actualDag)
-	Expect(dag[7][0].Name).To(Or(Equal("mount-base-overlay"), Equal("custom-mount")), actualDag)
-	Expect(dag[7][1].Name).To(Or(Equal("mount-base-overlay"), Equal("custom-mount")), actualDag)
-	Expect(dag[8][0].Name).To(Or(Equal("mount-bind"), Equal("overlay-mount")), actualDag)
-	Expect(dag[8][1].Name).To(Or(Equal("mount-bind"), Equal("overlay-mount")), actualDag)
-	Expect(dag[9][0].Name).To(Equal("write-fstab"), actualDag)
+	Expect(dag[2][0].Name).To(Equal(cnst.OpDiscoverState), actualDag)
+	Expect(dag[3][0].Name).To(Equal(cnst.OpMountRoot), actualDag)
+	Expect(dag[4][0].Name).To(Equal(cnst.OpMountOEM), actualDag)
+	Expect(dag[5][0].Name).To(Equal(cnst.OpRootfsHook), actualDag)
+	Expect(dag[6][0].Name).To(Equal(cnst.OpLoadConfig), actualDag)
+	Expect(dag[7][0].Name).To(Or(Equal(cnst.OpMountBaseOverlay), Equal(cnst.OpCustomMounts)), actualDag)
+	Expect(dag[7][1].Name).To(Or(Equal(cnst.OpMountBaseOverlay), Equal(cnst.OpCustomMounts)), actualDag)
+	Expect(dag[8][0].Name).To(Or(Equal(cnst.OpMountBind), Equal(cnst.OpOverlayMount)), actualDag)
+	Expect(dag[8][1].Name).To(Or(Equal(cnst.OpMountBind), Equal(cnst.OpOverlayMount)), actualDag)
+	Expect(dag[9][0].Name).To(Equal(cnst.OpWriteFstab), actualDag)
+	Expect(dag[10][0].Name).To(Equal(cnst.OpInitramfsHook), actualDag)
 }