From aa5939da89bc57eb396e0378ff0ea62d78232d2c Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 8 Mar 2023 11:45:11 +0100 Subject: [PATCH] Rework workflow (#77) This makes immucore run more in parallel rather than block everything else. We just tell sysroot.mount that eventually it will be mounted and to wait for a bit. This allows us to be more flexible where to run and run in parallel in cases like cdrom in which we may do things but we need the sysroot to be mounted already but not from us. Also adds the initramfs stage directly in immucore and merges all the dracut config into one Dont create sysroot, just add a timeout override so it waits for us Dont block on the service, just make sure to finish before initrd.target Fix mounts from cmdline More proper log Store logs under the /run/immucore dir Store rootfs and initramfs logs separated Do not log the full stages in INFO level Run initramfs stage in immucore directly on boot and cd/netboot Drop systemd requirement from dracut module Signed-off-by: Itxaka itxaka.garcia@spectrocloud.com --- Earthfile | 6 +- README.md | 2 + dracut/02-kairos-setup-initramfs.conf | 2 - dracut/10-immucore.conf | 9 +- dracut/28immucore/generator.sh | 32 ++- dracut/28immucore/immucore.service | 2 +- .../28immucore/kairos-setup-initramfs.service | 16 -- dracut/28immucore/module-setup.sh | 13 +- dracut/50-kairos-initrd.conf | 5 - internal/constants/constants.go | 2 + internal/utils/chroot.go | 191 ++++++++++++++++++ internal/utils/common.go | 1 - internal/utils/log.go | 5 +- internal/utils/mounts.go | 3 +- pkg/mount/dag_live_media.go | 8 + pkg/mount/dag_normal_boot.go | 5 + pkg/mount/dag_steps.go | 46 ++++- pkg/mount/dag_uki_boot.go | 2 +- pkg/mount/fs.go | 2 +- pkg/mount/operation.go | 18 +- pkg/mount/state.go | 88 +++++--- pkg/mount/state_test.go | 54 ++--- 22 files changed, 385 insertions(+), 127 deletions(-) delete mode 100644 dracut/02-kairos-setup-initramfs.conf delete mode 100644 dracut/28immucore/kairos-setup-initramfs.service delete mode 100644 dracut/50-kairos-initrd.conf create mode 100644 internal/utils/chroot.go diff --git a/Earthfile b/Earthfile index 48b4e10..69cbb80 100644 --- a/Earthfile +++ b/Earthfile @@ -73,10 +73,6 @@ dracut-artifacts: FROM $BASE_IMAGE WORKDIR /build COPY --dir dracut/28immucore . - COPY dracut/02-kairos-setup-initramfs.conf . COPY dracut/10-immucore.conf . - COPY dracut/50-kairos-initrd.conf . SAVE ARTIFACT 28immucore 28immucore - SAVE ARTIFACT 02-kairos-setup-initramfs.conf 02-kairos-setup-initramfs.conf - SAVE ARTIFACT 10-immucore.conf 10-immucore.conf - SAVE ARTIFACT 50-kairos-initrd.conf 50-kairos-initrd.conf \ No newline at end of file + SAVE ARTIFACT 10-immucore.conf 10-immucore.conf \ No newline at end of file diff --git a/README.md b/README.md index d8c4107..8ce415f 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,8 @@ There is also the `weak` value which indicates that this step has weak dependenc - `custom-mount`: This mounts the paths set in the config (`VOLUMES`) or in cmdline `rd.cos.mount=` in the given path (`LABEL=COS_PERSISTENT:/usr/local`) - `mount-bind`: This mounts the paths set in the config (`PERSISTENT_STATE_PATHS` and `CUSTOM_BIND_MOUNTS`) as bind mounts under the `PERSISTENT_STATE_TARGET` which defaults to `/usr/local/.state` - `write-fstab`: Writes the final fstab with all the mounts into `/sysroot/fstab` + - `initramfs-hook`: Runs the cloud config stage `initramfs`. Note that this is run under a chroot into what will be the final system (/sysroot). + - `wait-for-sysroot`: Waits for the /sysroot and /sysroot/system dirs to be available, which means that they are mounted. Useful when booting from CD/Netboot as immucore doesn't mount the /sysroot in those cases, but we want to run the initramfs stage once the system is ready. ### UKI mode (Experimental) diff --git a/dracut/02-kairos-setup-initramfs.conf b/dracut/02-kairos-setup-initramfs.conf deleted file mode 100644 index daa6e33..0000000 --- a/dracut/02-kairos-setup-initramfs.conf +++ /dev/null @@ -1,2 +0,0 @@ -install_items+=" /etc/hosts " -add_dracutmodules+=" network " diff --git a/dracut/10-immucore.conf b/dracut/10-immucore.conf index 4b9fa77..6a9fae1 100644 --- a/dracut/10-immucore.conf +++ b/dracut/10-immucore.conf @@ -1 +1,8 @@ -add_dracutmodules+=" immucore " \ No newline at end of file +hostonly_cmdline="no" +hostonly="no" +compress="xz" +i18n_install_all="yes" +show_modules="yes" +install_items+=" /etc/hosts " +omit_dracutmodules+=" multipath " +add_dracutmodules+=" livenet dmsquash-live immucore network " \ No newline at end of file diff --git a/dracut/28immucore/generator.sh b/dracut/28immucore/generator.sh index de90b9b..6e1db3d 100755 --- a/dracut/28immucore/generator.sh +++ b/dracut/28immucore/generator.sh @@ -8,25 +8,19 @@ GENERATOR_DIR="$2" [ -z "$GENERATOR_DIR" ] && exit 1 [ -d "$GENERATOR_DIR" ] || mkdir "$GENERATOR_DIR" - -## GENERATE SYSROOT -cos_img=$(getarg cos-img/filename=) -[ -z "${cos_img}" ] && exit 0 - -# This is necessary because otherwise systemd-fstab-generator will se the cmdline with the root=LABEL=X stanza and -# say, hey this is the ROOT where we need to boot! so it auto creates a sysroot.mount with the content of the value -# passed in the cmdline. But because we usually pass the label of the img (COS_ACTIVE) it will create the wrong mount -# service and be stuck in there forever. -# by generating it ourselves we get the sysroot.mount into the generators.early dir, which tells systemd to not generate it -# as it already exists and the rest is history +# Add a timeout to the sysroot so it waits a bit for immucore to mount it properly +mkdir -p "$GENERATOR_DIR"/sysroot.mount.d { - echo "[Unit]" - echo "Before=initrd-root-fs.target" - echo "DefaultDependencies=no" echo "[Mount]" - echo "Where=/sysroot" - echo "What=/run/initramfs/cos-state/${cos_img#/}" - echo "Options=ro,suid,dev,exec,auto,nouser,async" -} > "$GENERATOR_DIR"/sysroot.mount + echo "TimeoutSec=300" +} > "$GENERATOR_DIR"/sysroot.mount.d/timeout.conf -## END GENERATE SYSROOT \ No newline at end of file +# Make sure initrd-root-fs.target depends on sysroot.mount +# This seems to affect mainly ubuntu-22 where initrd-usr-fs depends on sysroot, but it has a broken link to it as sysroot.mount +# is generated under the generator.early dir but the link points to the generator dir. +# So it makes everything else a bit broken if you insert deps in the middle. +# By default other distros seem to do this as it shows on the map page https://man7.org/linux/man-pages/man7/dracut.bootup.7.html +if ! [ -L "$GENERATOR_DIR"/initrd-root-fs.target.wants/sysroot.mount ]; then + [ -d "$GENERATOR_DIR"/initrd-root-fs.target.wants ] || mkdir -p "$GENERATOR_DIR"/initrd-root-fs.target.wants + ln -s ../sysroot.mount "$GENERATOR_DIR"/initrd-root-fs.target.wants/sysroot.mount +fi \ No newline at end of file diff --git a/dracut/28immucore/immucore.service b/dracut/28immucore/immucore.service index b9fb631..bfb584e 100644 --- a/dracut/28immucore/immucore.service +++ b/dracut/28immucore/immucore.service @@ -3,7 +3,7 @@ Description=immucore DefaultDependencies=no After=systemd-udev-settle.service Requires=systemd-udev-settle.service -Before=dracut-initqueue.service sysroot.mount +Before=initrd-fs.target Conflicts=initrd-switch-root.target [Service] diff --git a/dracut/28immucore/kairos-setup-initramfs.service b/dracut/28immucore/kairos-setup-initramfs.service deleted file mode 100644 index 87cfb23..0000000 --- a/dracut/28immucore/kairos-setup-initramfs.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=kairos system initramfs setup before switch root -DefaultDependencies=no -After=initrd-fs.target -Requires=initrd-fs.target -Before=initrd.target - -[Service] -RootDirectory=/sysroot -BindPaths=/proc /sys /dev /run /tmp -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/elemental run-stage initramfs - -[Install] -RequiredBy=initrd.target diff --git a/dracut/28immucore/module-setup.sh b/dracut/28immucore/module-setup.sh index ecb6ecd..1d22f75 100755 --- a/dracut/28immucore/module-setup.sh +++ b/dracut/28immucore/module-setup.sh @@ -2,14 +2,12 @@ # called by dracut check() { - require_binaries "$systemdutildir"/systemd || return 1 - return 255 + return 0 } # called by dracut depends() { - echo systemd rootfs-block dm fs-lib - #tpm2-tss + echo rootfs-block dm fs-lib return 0 } @@ -31,13 +29,8 @@ install() { # missing mkfs.xfs xfs_growfs in image? inst_script "${moddir}/generator.sh" "${systemdutildir}/system-generators/immucore-generator" inst_simple "${moddir}/immucore.service" "${systemdsystemunitdir}/immucore.service" - mkdir -p "${initdir}/${systemdsystemunitdir}/initrd-fs.target.requires" - ln_r "../immucore.service" "${systemdsystemunitdir}/initrd-fs.target.requires/immucore.service" - - # Until this is done on immucore, we need to ship it as part of the dracut module - inst_simple "${moddir}/kairos-setup-initramfs.service" "${systemdsystemunitdir}/kairos-setup-initramfs.service" mkdir -p "${initdir}/${systemdsystemunitdir}/initrd.target.requires" - ln_r "../kairos-setup-initramfs.service" "${systemdsystemunitdir}/initrd.target.requires/kairos-setup-initramfs.service" + ln_r "../immucore.service" "${systemdsystemunitdir}/initrd.target.requires/immucore.service" dracut_need_initqueue } \ No newline at end of file diff --git a/dracut/50-kairos-initrd.conf b/dracut/50-kairos-initrd.conf deleted file mode 100644 index 81894b7..0000000 --- a/dracut/50-kairos-initrd.conf +++ /dev/null @@ -1,5 +0,0 @@ -hostonly_cmdline="no" -hostonly="no" -compress="xz" -omit_dracutmodules+=" multipath " -add_dracutmodules+=" livenet dmsquash-live " diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 28ed6c3..199d48b 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -27,5 +27,7 @@ const ( OpUkiInit = "uki-init" OpSentinel = "create-sentinel" OpUkiUdev = "uki-udev" + OpWaitForSysroot = "wait-for-sysroot" PersistentStateTarget = "/usr/local/.state" + LogDir = "/run/immucore" ) diff --git a/internal/utils/chroot.go b/internal/utils/chroot.go new file mode 100644 index 0000000..7a54c5f --- /dev/null +++ b/internal/utils/chroot.go @@ -0,0 +1,191 @@ +/* +Copyright © 2022 SUSE LLC +Copyright © 2023 Kairos authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +// Chroot represents the struct that will allow us to run commands inside a given chroot. +type Chroot struct { + path string + defaultMounts []string + activeMounts []string +} + +func NewChroot(path string) *Chroot { + return &Chroot{ + path: path, + defaultMounts: []string{"/dev", "/proc", "/sys", "/run", "/tmp"}, + activeMounts: []string{}, + } +} + +// ChrootedCallback runs the given callback in a chroot environment. +func ChrootedCallback(path string, bindMounts map[string]string, callback func() error) error { + chroot := NewChroot(path) + return chroot.RunCallback(callback) +} + +// Prepare will mount the defaultMounts as bind mounts, to be ready when we run chroot. +func (c *Chroot) Prepare() error { + var err error + + if len(c.activeMounts) > 0 { + return errors.New("there are already active mountpoints for this instance") + } + + defer func() { + if err != nil { + c.Close() + } + }() + + for _, mnt := range c.defaultMounts { + mountPoint := filepath.Join(c.path, mnt) + err = CreateIfNotExists(mountPoint) + if err != nil { + Log.Err(err).Str("what", mountPoint).Msg("Creating dir") + return err + } + err = syscall.Mount(mnt, mountPoint, "bind", syscall.MS_BIND|syscall.MS_REC, "") + if err != nil { + Log.Err(err).Str("where", mountPoint).Str("what", mnt).Msg("Mounting chroot bind") + return err + } + c.activeMounts = append(c.activeMounts, mountPoint) + } + + return nil +} + +// Close will unmount all active mounts created in Prepare on reverse order. +func (c *Chroot) Close() error { + failures := []string{} + for len(c.activeMounts) > 0 { + curr := c.activeMounts[len(c.activeMounts)-1] + Log.Debug().Str("what", curr).Msg("Unmounting from chroot") + c.activeMounts = c.activeMounts[:len(c.activeMounts)-1] + err := syscall.Unmount(curr, 0) + if err != nil { + Log.Err(err).Str("what", curr).Msg("Error unmounting") + failures = append(failures, curr) + } + } + if len(failures) > 0 { + c.activeMounts = failures + return fmt.Errorf("failed closing chroot environment. Unmount failures: %v", failures) + } + return nil +} + +// RunCallback runs the given callback in a chroot environment. +func (c *Chroot) RunCallback(callback func() error) (err error) { + var currentPath string + var oldRootF *os.File + + // Store current path + currentPath, err = os.Getwd() + if err != nil { + Log.Err(err).Msg("Failed to get current path") + return err + } + defer func() { + tmpErr := os.Chdir(currentPath) + if err == nil && tmpErr != nil { + err = tmpErr + } + }() + + // Store current root + oldRootF, err = os.Open("/") + if err != nil { + Log.Err(err).Msg("Can't open current root") + return err + } + defer oldRootF.Close() + + if len(c.activeMounts) == 0 { + err = c.Prepare() + if err != nil { + Log.Err(err).Msg("Can't mount default mounts") + return err + } + defer func() { + tmpErr := c.Close() + if err == nil { + err = tmpErr + } + }() + } + // Change to new dir before running chroot! + err = syscall.Chdir(c.path) + if err != nil { + Log.Err(err).Str("path", c.path).Msg("Can't chdir") + return err + } + + err = syscall.Chroot(c.path) + if err != nil { + Log.Err(err).Str("path", c.path).Msg("Can't chroot") + return err + } + + // Restore to old root + defer func() { + tmpErr := oldRootF.Chdir() + if tmpErr != nil { + Log.Err(tmpErr).Str("path", oldRootF.Name()).Msg("Can't change to old root dir") + if err == nil { + err = tmpErr + } + } else { + tmpErr = syscall.Chroot(".") + if tmpErr != nil { + Log.Err(tmpErr).Str("path", oldRootF.Name()).Msg("Can't chroot back to old root") + if err == nil { + err = tmpErr + } + } + } + }() + + return callback() +} + +// Run executes a command inside a chroot. +func (c *Chroot) Run(command string) (string, error) { + var err error + var out []byte + callback := func() error { + cmd := exec.Command("/bin/sh", "-c", command) + cmd.Env = os.Environ() + out, err = cmd.CombinedOutput() + return err + } + err = c.RunCallback(callback) + if err != nil { + Log.Err(err).Str("cmd", command).Msg("Cant run command on chroot") + } + return string(out), err +} diff --git a/internal/utils/common.go b/internal/utils/common.go index e133c1c..5260f43 100644 --- a/internal/utils/common.go +++ b/internal/utils/common.go @@ -182,7 +182,6 @@ func CommandWithPath(c string) (string, error) { pathAppend = fmt.Sprintf("%s:%s", pathAppend, splitted[1]) } } - Log.Debug().Str("content", pathAppend).Msg("PATH") cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", pathAppend)) o, err := cmd.CombinedOutput() return string(o), err diff --git a/internal/utils/log.go b/internal/utils/log.go index 49665f2..978f1ad 100644 --- a/internal/utils/log.go +++ b/internal/utils/log.go @@ -3,7 +3,9 @@ package utils import ( "io" "os" + "path/filepath" + "github.com/kairos-io/immucore/internal/constants" "github.com/rs/zerolog" ) @@ -16,7 +18,8 @@ func CloseLogFiles() { func SetLogger() { var loggers []io.Writer - logFile, err := os.Create("/run/immucore.log") + _ = os.MkdirAll(constants.LogDir, os.ModeDir|os.ModePerm) + logFile, err := os.Create(filepath.Join(constants.LogDir, "immucore.log")) if err == nil { loggers = append(loggers, zerolog.ConsoleWriter{Out: logFile}) } diff --git a/internal/utils/mounts.go b/internal/utils/mounts.go index a2b22fd..a9134f2 100644 --- a/internal/utils/mounts.go +++ b/internal/utils/mounts.go @@ -169,9 +169,8 @@ func Fsck(device string) error { cmd := strings.Join(args, " ") Log.Debug().Str("cmd", cmd).Msg("fsck command") out, e := CommandWithPath(cmd) - Log.Debug().Str("output", out).Msg("fsck output") if e != nil { - Log.Warn().Str("error", e.Error()).Str("what", device).Msg("fsck") + Log.Debug().Err(e).Str("out", out).Str("what", device).Msg("fsck") } return e } diff --git a/pkg/mount/dag_live_media.go b/pkg/mount/dag_live_media.go index 263a6ce..57325d5 100644 --- a/pkg/mount/dag_live_media.go +++ b/pkg/mount/dag_live_media.go @@ -1,6 +1,7 @@ package mount import ( + cnst "github.com/kairos-io/immucore/internal/constants" "github.com/spectrocloud-labs/herd" ) @@ -9,5 +10,12 @@ import ( func (s *State) RegisterLiveMedia(g *herd.Graph) error { // Maybe LogIfErrorAndPanic ? If no sentinel, a lot of config files are not going to run err := s.LogIfErrorAndReturn(s.WriteSentinelDagStep(g), "write sentinel") + + // Waits for sysroot to be there, just in case + s.LogIfError(s.WaitForSysrootDagStep(g), "Waiting for sysroot") + // Run rootfs + s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel, cnst.OpWaitForSysroot), "rootfs stage") + // Run initramfs inside the /sysroot chroot! + s.LogIfError(s.InitramfsStageDagStep(g, herd.WithDeps(cnst.OpSentinel, cnst.OpWaitForSysroot, cnst.OpRootfsHook), herd.WithWeakDeps()), "initramfs stage") return err } diff --git a/pkg/mount/dag_normal_boot.go b/pkg/mount/dag_normal_boot.go index 3840388..b96548f 100644 --- a/pkg/mount/dag_normal_boot.go +++ b/pkg/mount/dag_normal_boot.go @@ -51,5 +51,10 @@ func (s *State) RegisterNormalBoot(g *herd.Graph) error { // Write fstab file s.LogIfError(s.WriteFstabDagStep(g), "write fstab") + // do it after fstab is created + s.LogIfError(s.InitramfsStageDagStep(g, + herd.WithDeps(cnst.OpMountRoot, cnst.OpDiscoverState, cnst.OpLoadConfig, cnst.OpWriteFstab), + herd.WithWeakDeps(cnst.OpMountBaseOverlay, cnst.OpMountOEM, cnst.OpMountBind, cnst.OpMountBind, cnst.OpCustomMounts, cnst.OpOverlayMount), + ), "initramfs stage") return err } diff --git a/pkg/mount/dag_steps.go b/pkg/mount/dag_steps.go index 06683ac..1047d39 100644 --- a/pkg/mount/dag_steps.go +++ b/pkg/mount/dag_steps.go @@ -107,8 +107,8 @@ func (s *State) RootfsStageDagStep(g *herd.Graph, deps ...string) error { } // InitramfsStageDagStep will add the rootfs stage. -func (s *State) InitramfsStageDagStep(g *herd.Graph, deps ...string) error { - return g.Add(cnst.OpInitramfsHook, herd.WithDeps(deps...), herd.WeakDeps, herd.WithCallback(s.RunStageOp("initramfs"))) +func (s *State) InitramfsStageDagStep(g *herd.Graph, deps herd.OpOption, weakDeps herd.OpOption) error { + return g.Add(cnst.OpInitramfsHook, deps, weakDeps, herd.WithCallback(s.RunStageOp("initramfs"))) } // LoadEnvLayoutDagStep will add the stage to load from cos-layout.env and fill the proper CustomMounts, OverlayDirs and BindMounts. @@ -163,11 +163,8 @@ func (s *State) LoadEnvLayoutDagStep(g *herd.Graph, deps ...string) error { // Parse custom mounts also from cmdline (rd.cos.mount=) // Parse custom mounts also from cmdline (rd.immucore.mount=) // Parse custom mounts also from env file (VOLUMES) - var mounts []string - mounts = internalUtils.CleanupSlice(internalUtils.ReadCMDLineArg("rd.cos.mount=")) - mounts = append(mounts, internalUtils.CleanupSlice(internalUtils.ReadCMDLineArg("rd.immucore.mount="))...) - mounts = append(mounts, env["VOLUMES"]) - for _, v := range mounts { + + for _, v := range append(append(internalUtils.ReadCMDLineArg("rd.cos.mount="), internalUtils.ReadCMDLineArg("rd.immucore.mount=")...), strings.Split(env["VOLUMES"], " ")...) { addLine(internalUtils.ParseMount(v)) } @@ -257,6 +254,7 @@ func (s *State) MountCustomMountsDagStep(g *herd.Graph) error { herd.WithDeps(cnst.OpLoadConfig), herd.WithCallback(func(ctx context.Context) error { var err *multierror.Error + internalUtils.Log.Debug().Interface("mounts", s.CustomMounts).Msg("Mounting custom mounts") for what, where := range s.CustomMounts { // TODO: scan for the custom mount disk to know the underlying fs and set it proper @@ -471,3 +469,37 @@ func (s *State) LoadKernelModules(g *herd.Graph) error { }), ) } + +// WaitForSysrootDagStep waits for the s.Rootdir and s.Rootdir/system paths to be there +// Useful for livecd/netboot as we want to run steps after s.Rootdir is ready but we don't mount it ourselves. +func (s *State) WaitForSysrootDagStep(g *herd.Graph) error { + return g.Add(cnst.OpWaitForSysroot, + herd.WithCallback(func(ctx context.Context) error { + cc := time.After(60 * time.Second) + for { + select { + default: + time.Sleep(2 * time.Second) + _, err := os.Stat(s.Rootdir) + if err != nil { + internalUtils.Log.Debug().Str("what", s.Rootdir).Msg("Checking path existence") + continue + } + _, err = os.Stat(filepath.Join(s.Rootdir, "system")) + if err != nil { + internalUtils.Log.Debug().Str("what", filepath.Join(s.Rootdir, "system")).Msg("Checking path existence") + continue + } + return nil + case <-ctx.Done(): + e := fmt.Errorf("context canceled") + internalUtils.Log.Err(e).Str("what", s.Rootdir).Msg("filepath check canceled") + return e + case <-cc: + e := fmt.Errorf("timeout exhausted") + internalUtils.Log.Err(e).Str("what", s.Rootdir).Msg("filepath check timeout") + return e + } + } + })) +} diff --git a/pkg/mount/dag_uki_boot.go b/pkg/mount/dag_uki_boot.go index 47f610b..ee04db4 100644 --- a/pkg/mount/dag_uki_boot.go +++ b/pkg/mount/dag_uki_boot.go @@ -39,7 +39,7 @@ func (s *State) RegisterUKI(g *herd.Graph) error { s.LogIfError(s.MountCustomBindsDagStep(g), "custom binds mount") // run initramfs stage - s.LogIfError(s.InitramfsStageDagStep(g, cnst.OpMountBind), "uki initramfs") + s.LogIfError(s.InitramfsStageDagStep(g, herd.WithDeps(cnst.OpMountBind), herd.WithWeakDeps()), "uki initramfs") s.LogIfError(g.Add(cnst.OpWriteFstab, herd.WithDeps(cnst.OpLoadConfig, cnst.OpCustomMounts, cnst.OpMountBind, cnst.OpOverlayMount), diff --git a/pkg/mount/fs.go b/pkg/mount/fs.go index 83c3f7c..4fa3c37 100644 --- a/pkg/mount/fs.go +++ b/pkg/mount/fs.go @@ -80,7 +80,7 @@ func mountBind(mountpoint, root, stateTarget string) mountOperation { "bind", }, } - internalUtils.Log.Debug().Str("mountpoint", mountpoint).Str("root", root).Str("bindMountPath", bindMountPath).Msg("BIND") + internalUtils.Log.Debug().Str("where", rootMount).Str("what", stateDir).Msg("Bind mount") tmpFstab := internalUtils.MountToFstab(tmpMount) tmpFstab.File = internalUtils.CleanSysrootForFstab(fmt.Sprintf("/%s", mountpoint)) tmpFstab.Spec = internalUtils.CleanSysrootForFstab(tmpFstab.Spec) diff --git a/pkg/mount/operation.go b/pkg/mount/operation.go index 5bda5bd..2398644 100644 --- a/pkg/mount/operation.go +++ b/pkg/mount/operation.go @@ -6,6 +6,7 @@ import ( "github.com/kairos-io/immucore/internal/constants" internalUtils "github.com/kairos-io/immucore/internal/utils" "github.com/moby/sys/mountinfo" + "github.com/rs/zerolog" ) type mountOperation struct { @@ -16,23 +17,30 @@ type mountOperation struct { } func (m mountOperation) run() error { - internalUtils.Log.With().Str("what", m.MountOption.Source).Str("where", m.Target).Str("type", m.MountOption.Type).Strs("options", m.MountOption.Options).Logger() + // Add context to sublogger + l := internalUtils.Log.With().Str("what", m.MountOption.Source).Str("where", m.Target).Str("type", m.MountOption.Type).Strs("options", m.MountOption.Options).Logger().Level(zerolog.InfoLevel) + // Not sure why this defaults to debuglevel when creating a sublogger, so make sure we set it properly + debug := len(internalUtils.ReadCMDLineArg("rd.immucore.debug")) > 0 + if debug { + l.Level(zerolog.DebugLevel) + } + if m.PrepareCallback != nil { if err := m.PrepareCallback(); err != nil { - internalUtils.Log.Err(err).Msg("executing mount callback") + l.Err(err).Msg("executing mount callback") return err } } //TODO: not only check if mounted but also if the type,options and source are the same? mounted, err := mountinfo.Mounted(m.Target) if err != nil { - internalUtils.Log.Err(err).Msg("checking mount status") + l.Err(err).Msg("checking mount status") return err } if mounted { - internalUtils.Log.Debug().Msg("Already mounted") + l.Debug().Msg("Already mounted") return constants.ErrAlreadyMounted } - internalUtils.Log.Debug().Msg("mount ready") + l.Debug().Msg("mount ready") return mount.All([]mount.Mount{m.MountOption}, m.Target) } diff --git a/pkg/mount/state.go b/pkg/mount/state.go index 195ec27..629f7a7 100644 --- a/pkg/mount/state.go +++ b/pkg/mount/state.go @@ -41,6 +41,12 @@ func (s *State) path(p ...string) string { func (s *State) WriteFstab(fstabFile string) func(context.Context) error { return func(ctx context.Context) error { + // Create the file first, override if something is there, we don't care, we are on initramfs + f, err := os.Create(fstabFile) + if err != nil { + return err + } + f.Close() for _, fst := range s.fstabs { select { case <-ctx.Done(): @@ -65,35 +71,63 @@ func (s *State) WriteFstab(fstabFile string) func(context.Context) error { // If its uki we don't symlink as we already have everything in the sysroot. func (s *State) RunStageOp(stage string) func(context.Context) error { return func(ctx context.Context) error { - if stage == "rootfs" && !internalUtils.IsUKI() { - if _, err := os.Stat("/system"); os.IsNotExist(err) { - err = os.Symlink("/sysroot/system", "/system") - if err != nil { - internalUtils.Log.Err(err).Msg("creating symlink") - } - } - if _, err := os.Stat("/oem"); os.IsNotExist(err) { - err = os.Symlink("/sysroot/oem", "/oem") - if err != nil { - internalUtils.Log.Err(err).Msg("creating symlink") - } - } - } - - cmd := fmt.Sprintf("/usr/bin/elemental run-stage %s", stage) + cmd := fmt.Sprintf("elemental run-stage %s", stage) // If we set the level to debug, also call elemental with debug if internalUtils.Log.GetLevel() == zerolog.DebugLevel { cmd = fmt.Sprintf("%s --debug", cmd) } - output, err := utils.SH(cmd) - internalUtils.Log.Debug().Msg(output) - return err + + switch stage { + case "rootfs": + if !internalUtils.IsUKI() { + if _, err := os.Stat("/system"); os.IsNotExist(err) { + err = os.Symlink("/sysroot/system", "/system") + if err != nil { + internalUtils.Log.Err(err).Msg("creating symlink") + } + } + if _, err := os.Stat("/oem"); os.IsNotExist(err) { + err = os.Symlink("/sysroot/oem", "/oem") + if err != nil { + internalUtils.Log.Err(err).Msg("creating symlink") + } + } + } + output, err := utils.SH(cmd) + internalUtils.Log.Info().Msg("Running rootfs stage") + internalUtils.Log.Info().Msg(output) + f, ferr := os.Create(filepath.Join(constants.LogDir, "rootfs_stage.log")) + if ferr == nil { + _, _ = f.WriteString(output) + _ = f.Close() + } + return err + case "initramfs": + // Not sure if it will work under UKI where the s.Rootdir is the current root already + chroot := internalUtils.NewChroot(s.Rootdir) + output, err := chroot.Run(cmd) + internalUtils.Log.Info().Msg("Running initramfs stage") + internalUtils.Log.Info().Msg(output) + f, ferr := os.Create(filepath.Join(constants.LogDir, "initramfs_stage.log")) + if ferr == nil { + _, _ = f.WriteString(output) + _ = f.Close() + } + return err + default: + return errors.New("no stage that we know off") + } } } // MountOP creates and executes a mount operation. func (s *State) MountOP(what, where, t string, options []string, timeout time.Duration) func(context.Context) error { - internalUtils.Log.With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger() + l := internalUtils.Log.With().Str("what", what).Str("where", where).Str("type", t).Strs("options", options).Logger().Level(zerolog.InfoLevel) + // Not sure why this defaults to debuglevel when creating a sublogger, so make sure we set it properly + debug := len(internalUtils.ReadCMDLineArg("rd.immucore.debug")) > 0 + if debug { + l.Level(zerolog.DebugLevel) + } return func(c context.Context) error { cc := time.After(timeout) @@ -102,7 +136,7 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du default: err := internalUtils.CreateIfNotExists(where) if err != nil { - internalUtils.Log.Err(err).Msg("Creating dir") + l.Err(err).Msg("Creating dir") continue } time.Sleep(1 * time.Second) @@ -131,18 +165,18 @@ func (s *State) MountOP(what, where, t string, options []string, timeout time.Du // only continue the loop if it's an error and not an already mounted error if err != nil && !errors.Is(err, constants.ErrAlreadyMounted) { - internalUtils.Log.Err(err).Send() + l.Err(err).Send() continue } - internalUtils.Log.Debug().Msg("mount done") + l.Info().Msg("mount done") return nil case <-c.Done(): e := fmt.Errorf("context canceled") - internalUtils.Log.Err(e).Msg("mount canceled") + l.Err(e).Msg("mount canceled") return e case <-cc: e := fmt.Errorf("timeout exhausted") - internalUtils.Log.Err(e).Msg("Mount timeout") + l.Err(e).Msg("Mount timeout") return e } } @@ -155,9 +189,9 @@ func (s *State) WriteDAG(g *herd.Graph) (out string) { out += fmt.Sprintf("%d.\n", i+1) for _, op := range layer { if op.Error != nil { - out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps) + out += fmt.Sprintf(" <%s> (error: %s) (background: %t) (weak: %t) (run: %t)\n", op.Name, op.Error.Error(), op.Background, op.WeakDeps, op.Executed) } else { - out += fmt.Sprintf(" <%s> (background: %t) (weak: %t)\n", op.Name, op.Background, op.WeakDeps) + out += fmt.Sprintf(" <%s> (background: %t) (weak: %t) (run: %t)\n", op.Name, op.Background, op.WeakDeps, op.Executed) } } } diff --git a/pkg/mount/state_test.go b/pkg/mount/state_test.go index 9c7a939..6f52ad6 100644 --- a/pkg/mount/state_test.go +++ b/pkg/mount/state_test.go @@ -2,6 +2,7 @@ package mount_test import ( "context" + cnst "github.com/kairos-io/immucore/internal/constants" "time" "github.com/kairos-io/immucore/pkg/mount" @@ -67,16 +68,21 @@ var _ = Describe("mounting immutable setup", func() { }) func checkLiveCDDag(dag [][]herd.GraphEntry, actualDag string) { - Expect(len(dag)).To(Equal(2), actualDag) + Expect(len(dag)).To(Equal(4), actualDag) Expect(len(dag[0])).To(Equal(1), actualDag) - Expect(len(dag[1])).To(Equal(1), actualDag) + Expect(len(dag[1])).To(Equal(2), actualDag) + Expect(len(dag[2])).To(Equal(1), actualDag) + Expect(len(dag[3])).To(Equal(1), actualDag) Expect(dag[0][0].Name).To(Equal("init")) - Expect(dag[1][0].Name).To(Equal("create-sentinel")) + Expect(dag[1][0].Name).To(Or(Equal(cnst.OpSentinel), Equal(cnst.OpWaitForSysroot)), actualDag) + Expect(dag[1][1].Name).To(Or(Equal(cnst.OpSentinel), Equal(cnst.OpWaitForSysroot)), actualDag) + Expect(dag[2][0].Name).To(Equal(cnst.OpRootfsHook), actualDag) + Expect(dag[3][0].Name).To(Equal(cnst.OpInitramfsHook), actualDag) } func checkDag(dag [][]herd.GraphEntry, actualDag string) { - Expect(len(dag)).To(Equal(10), actualDag) + Expect(len(dag)).To(Equal(11), actualDag) Expect(len(dag[0])).To(Equal(1), actualDag) Expect(len(dag[1])).To(Equal(3), actualDag) Expect(len(dag[2])).To(Equal(1), actualDag) @@ -87,31 +93,33 @@ func checkDag(dag [][]herd.GraphEntry, actualDag string) { Expect(len(dag[7])).To(Equal(2), actualDag) Expect(len(dag[8])).To(Equal(2), actualDag) Expect(len(dag[9])).To(Equal(1), actualDag) + Expect(len(dag[10])).To(Equal(1), actualDag) Expect(dag[0][0].Name).To(Equal("init")) Expect(dag[1][0].Name).To(Or( - Equal("mount-tmpfs"), - Equal("create-sentinel"), - Equal("mount-state"), + Equal(cnst.OpMountTmpfs), + Equal(cnst.OpSentinel), + Equal(cnst.OpMountState), ), actualDag) Expect(dag[1][1].Name).To(Or( - Equal("mount-tmpfs"), - Equal("create-sentinel"), - Equal("mount-state"), + Equal(cnst.OpMountTmpfs), + Equal(cnst.OpSentinel), + Equal(cnst.OpMountState), ), actualDag) Expect(dag[1][2].Name).To(Or( - Equal("mount-tmpfs"), - Equal("create-sentinel"), - Equal("mount-state"), + Equal(cnst.OpMountTmpfs), + Equal(cnst.OpSentinel), + Equal(cnst.OpMountState), ), actualDag) - Expect(dag[2][0].Name).To(Equal("discover-state"), actualDag) - Expect(dag[3][0].Name).To(Equal("mount-root"), actualDag) - Expect(dag[4][0].Name).To(Equal("mount-oem"), actualDag) - Expect(dag[5][0].Name).To(Equal("rootfs-hook"), actualDag) - Expect(dag[6][0].Name).To(Equal("load-config"), actualDag) - Expect(dag[7][0].Name).To(Or(Equal("mount-base-overlay"), Equal("custom-mount")), actualDag) - Expect(dag[7][1].Name).To(Or(Equal("mount-base-overlay"), Equal("custom-mount")), actualDag) - Expect(dag[8][0].Name).To(Or(Equal("mount-bind"), Equal("overlay-mount")), actualDag) - Expect(dag[8][1].Name).To(Or(Equal("mount-bind"), Equal("overlay-mount")), actualDag) - Expect(dag[9][0].Name).To(Equal("write-fstab"), actualDag) + Expect(dag[2][0].Name).To(Equal(cnst.OpDiscoverState), actualDag) + Expect(dag[3][0].Name).To(Equal(cnst.OpMountRoot), actualDag) + Expect(dag[4][0].Name).To(Equal(cnst.OpMountOEM), actualDag) + Expect(dag[5][0].Name).To(Equal(cnst.OpRootfsHook), actualDag) + Expect(dag[6][0].Name).To(Equal(cnst.OpLoadConfig), actualDag) + Expect(dag[7][0].Name).To(Or(Equal(cnst.OpMountBaseOverlay), Equal(cnst.OpCustomMounts)), actualDag) + Expect(dag[7][1].Name).To(Or(Equal(cnst.OpMountBaseOverlay), Equal(cnst.OpCustomMounts)), actualDag) + Expect(dag[8][0].Name).To(Or(Equal(cnst.OpMountBind), Equal(cnst.OpOverlayMount)), actualDag) + Expect(dag[8][1].Name).To(Or(Equal(cnst.OpMountBind), Equal(cnst.OpOverlayMount)), actualDag) + Expect(dag[9][0].Name).To(Equal(cnst.OpWriteFstab), actualDag) + Expect(dag[10][0].Name).To(Equal(cnst.OpInitramfsHook), actualDag) }