Bring UKI to a working state (#97)

- Mount the needed base mounts (/proc /dev /sys /tmp)
 - Use our own console for yip (required to add the PATH under uki)
 - Order the DAG in a proper way (was out of order and not working)

Signed-off-by: Itxaka <itxaka.garcia@spectrocloud.com>

internal/constants/constants.go

@@ -31,6 +31,8 @@ const (
 	OpUkiInit             = "uki-init"
 	OpSentinel            = "create-sentinel"
 	OpUkiUdev             = "uki-udev"
+	OpUkiBaseMounts       = "uki-base-mounts"
+	OpUkiKernelModules    = "uki-kernel-modules"
 	OpWaitForSysroot      = "wait-for-sysroot"
 	PersistentStateTarget = "/usr/local/.state"
 	LogDir                = "/run/immucore"

internal/utils/cloudinit.go

@@ -56,9 +56,9 @@ func RunStage(stage string) (bytes.Buffer, error) {
 	var buffer bytes.Buffer
 	log := logrus.New()
 	log.SetOutput(&buffer)
-	log.SetLevel(logrus.DebugLevel)
+	log.SetLevel(logrus.InfoLevel)
 	yip := NewYipExecutor(log)
-	c := console.NewStandardConsole(console.WithLogger(log))
+	c := ImmucoreConsole{}
 
 	stageBefore := fmt.Sprintf("%s.before", stage)
 	stageAfter := fmt.Sprintf("%s.after", stage)

internal/utils/cloudinit_console.go

@@ -0,0 +1,48 @@
+package utils
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// ImmucoreConsole is the console for yip. As we have to hijack the Run method to be able to run under UKI
+// To add the paths, we need to create our own console.
+type ImmucoreConsole struct {
+}
+
+func (s ImmucoreConsole) Run(cmd string, opts ...func(cmd *exec.Cmd)) (string, error) {
+	c := PrepareCommandWithPath(cmd)
+	for _, o := range opts {
+		o(c)
+	}
+	out, err := c.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("failed to run %s: %v", cmd, err)
+	}
+
+	return string(out), err
+}
+
+func (s ImmucoreConsole) Start(cmd *exec.Cmd, opts ...func(cmd *exec.Cmd)) error {
+	for _, o := range opts {
+		o(cmd)
+	}
+	return cmd.Run()
+}
+
+func (s ImmucoreConsole) RunTemplate(st []string, template string) error {
+	var errs error
+
+	for _, svc := range st {
+		out, err := s.Run(fmt.Sprintf(template, svc))
+		if err != nil {
+			Log.Err(err).Msg("Run template")
+			Log.Debug().Str("output", out).Msg("Run template")
+			errs = multierror.Append(errs, err)
+			continue
+		}
+	}
+	return errs
+}

internal/utils/common.go

@@ -191,6 +191,23 @@ func CommandWithPath(c string) (string, error) {
 	return string(o), err
 }
 
+// PrepareCommandWithPath prepares a cmd with the proper env
+// For running under yip.
+func PrepareCommandWithPath(c string) *exec.Cmd {
+	cmd := exec.Command("/bin/sh", "-c", c)
+	cmd.Env = os.Environ()
+	pathAppend := "/usr/bin:/usr/sbin:/bin:/sbin"
+	// try to extract any existing path from the environment
+	for _, env := range cmd.Env {
+		splitted := strings.Split(env, "=")
+		if splitted[0] == "PATH" {
+			pathAppend = fmt.Sprintf("%s:%s", pathAppend, splitted[1])
+		}
+	}
+	cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", pathAppend))
+	return cmd
+}
+
 // GetHostProcCmdline returns the path to /proc/cmdline
 // Mainly used to override the cmdline during testing.
 func GetHostProcCmdline() string {
@@ -211,7 +228,6 @@ func GetPartByLabel(label string, attempts int) (string, error) {
 		}
 		for _, d := range blockDevices.Disks {
 			for _, part := range d.Partitions {
-				Log.Info().Interface("part", part).Str("label", label).Msg("getpartbylabel")
 				if part.FilesystemLabel == label {
 					return filepath.Join("/dev/", part.Name), nil
 				}

pkg/mount/dag_steps.go

@@ -330,8 +330,9 @@ func (s *State) WriteFstabDagStep(g *herd.Graph) error {
 
 // WriteSentinelDagStep sets the sentinel file to identify the boot mode.
 // This is used by several things to know in which state they are, for example cloud configs.
-func (s *State) WriteSentinelDagStep(g *herd.Graph) error {
+func (s *State) WriteSentinelDagStep(g *herd.Graph, deps ...string) error {
 	return g.Add(cnst.OpSentinel,
+		herd.WithDeps(deps...),
 		herd.WithCallback(func(ctx context.Context) error {
 			var sentinel string
 
@@ -376,13 +377,75 @@ func (s *State) WriteSentinelDagStep(g *herd.Graph) error {
 		}))
 }
 
+func (s *State) UKIMountBaseSystem(g *herd.Graph) error {
+	type mount struct {
+		where string
+		what  string
+		fs    string
+		flags uintptr
+		data  string
+	}
+
+	return g.Add(
+		cnst.OpUkiBaseMounts,
+		herd.WithCallback(
+			func(ctx context.Context) error {
+				var err error
+				mounts := []mount{
+					{
+						"/run",
+						"tmpfs",
+						"tmpfs",
+						syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_RELATIME,
+						"mode=755",
+					},
+					{
+						"/sys",
+						"sysfs",
+						"sysfs",
+						syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_RELATIME,
+						"",
+					},
+					{
+						"/dev",
+						"devtmpfs",
+						"devtmpfs",
+						syscall.MS_NOSUID,
+						"mode=755",
+					},
+					{
+						"/tmp",
+						"tmpfs",
+						"tmpfs",
+						syscall.MS_NOSUID | syscall.MS_NODEV,
+						"",
+					},
+				}
+				for _, m := range mounts {
+					e := os.MkdirAll(m.where, 0755)
+					if e != nil {
+						err = multierror.Append(err, e)
+						internalUtils.Log.Err(e).Msg("Creating dir")
+					}
+					e = syscall.Mount(m.what, m.where, m.fs, m.flags, m.data)
+					if e != nil {
+						err = multierror.Append(err, e)
+						internalUtils.Log.Err(e).Str("what", m.what).Str("where", m.where).Str("type", m.fs).Msg("Mounting")
+					}
+				}
+				return err
+			},
+		),
+	)
+}
+
 // UKIBootInitDagStep tries to launch /sbin/init in root and pass over the system
 // booting to the real init process
 // Drops to emergency if not able to. Panic if it cant even launch emergency.
-func (s *State) UKIBootInitDagStep(g *herd.Graph, deps ...string) error {
+func (s *State) UKIBootInitDagStep(g *herd.Graph) error {
 	return g.Add(cnst.OpUkiInit,
-		herd.WithDeps(deps...),
-		herd.WeakDeps,
+		herd.WithDeps(),
+		herd.WithWeakDeps(cnst.OpRemountRootRO, cnst.OpRootfsHook, cnst.OpInitramfsHook, cnst.OpWriteFstab),
 		herd.WithCallback(func(ctx context.Context) error {
 			// Print dag before exit, otherwise its never printed as we never exit the program
 			internalUtils.Log.Info().Msg(s.WriteDAG(g))
@@ -400,11 +463,20 @@ func (s *State) UKIBootInitDagStep(g *herd.Graph, deps ...string) error {
 }
 
 // UKIRemountRootRODagStep remount root read only.
-func (s *State) UKIRemountRootRODagStep(g *herd.Graph, deps ...string) error {
+func (s *State) UKIRemountRootRODagStep(g *herd.Graph) error {
 	return g.Add(cnst.OpRemountRootRO,
-		herd.WithDeps(deps...),
+		herd.WithDeps(cnst.OpRootfsHook),
 		herd.WithCallback(func(ctx context.Context) error {
-			return syscall.Mount("/", "/", "rootfs", syscall.MS_REMOUNT|syscall.MS_RDONLY, "")
+			var err error
+			for i := 1; i < 5; i++ {
+				time.Sleep(1 * time.Second)
+				// Should we try to stop udev here?
+				err = syscall.Mount("", "/", "", syscall.MS_REMOUNT|syscall.MS_RDONLY, "")
+				if err != nil {
+					continue
+				}
+			}
+			return err
 		}),
 	)
 }
@@ -413,6 +485,7 @@ func (s *State) UKIRemountRootRODagStep(g *herd.Graph, deps ...string) error {
 // Needed if we expect to find devices by label...
 func (s *State) UKIUdevDaemon(g *herd.Graph) error {
 	return g.Add(cnst.OpUkiUdev,
+		herd.WithDeps(cnst.OpUkiBaseMounts, cnst.OpUkiKernelModules),
 		herd.WithCallback(func(ctx context.Context) error {
 			// Should probably figure out other udevd binaries....
 			var udevBin string
@@ -448,7 +521,8 @@ func (s *State) UKIUdevDaemon(g *herd.Graph) error {
 // Mainly block devices and net devices
 // probably others down the line.
 func (s *State) LoadKernelModules(g *herd.Graph) error {
-	return g.Add("kernel-modules",
+	return g.Add(cnst.OpUkiKernelModules,
+		herd.WithDeps(cnst.OpUkiBaseMounts),
 		herd.WithCallback(func(ctx context.Context) error {
 			drivers, err := kdetect.ProbeKernelModules("")
 			if err != nil {

pkg/mount/dag_uki_boot.go

@@ -7,27 +7,31 @@ import (
 
 // RegisterUKI registers the dag for booting from UKI.
 func (s *State) RegisterUKI(g *herd.Graph) error {
-	// Write sentinel
-	s.LogIfError(s.WriteSentinelDagStep(g), "sentinel")
+	// Mount basic mounts
+	s.LogIfError(s.UKIMountBaseSystem(g), "mounting base mounts")
 
+	// Write sentinel
+	s.LogIfError(s.WriteSentinelDagStep(g, cnst.OpUkiBaseMounts), "sentinel")
+
+	// Load needed kernel modules
 	s.LogIfError(s.LoadKernelModules(g), "kernel modules")
 
 	// Udev for devices discovery
 	s.LogIfError(s.UKIUdevDaemon(g), "udev")
 
 	// Run rootfs stage
-	s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel), "uki rootfs")
+	s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel, cnst.OpUkiUdev), "uki rootfs")
 
 	// Remount root RO
-	s.LogIfError(s.UKIRemountRootRODagStep(g, cnst.OpRootfsHook), "remount root")
-
-	// Mount base overlay under /run/overlay
-	s.LogIfError(s.MountBaseOverlayDagStep(g), "base overlay")
+	s.LogIfError(s.UKIRemountRootRODagStep(g), "remount root")
 
 	// Populate state bind mounts, overlay mounts, custom-mounts from /run/cos/cos-layout.env
 	// Requires stage rootfs to have run, which usually creates the cos-layout.env file
 	s.LogIfError(s.LoadEnvLayoutDagStep(g, cnst.OpRootfsHook), "loading cos-layout.env")
 
+	// Mount base overlay under /run/overlay
+	s.LogIfError(s.MountBaseOverlayDagStep(g), "base overlay")
+
 	// Mount custom overlays loaded from the /run/cos/cos-layout.env file
 	s.LogIfError(s.MountCustomOverlayDagStep(g), "custom overlays mount")
 
@@ -47,6 +51,6 @@ func (s *State) RegisterUKI(g *herd.Graph) error {
 		herd.WithCallback(s.WriteFstab(s.path("/etc/fstab")))), "fstab")
 
 	// Handover to /sbin/init
-	_ = s.UKIBootInitDagStep(g, cnst.OpRemountRootRO, cnst.OpRootfsHook, cnst.OpInitramfsHook)
+	_ = s.UKIBootInitDagStep(g)
 	return nil
 }

pkg/mount/state.go

@@ -85,7 +85,12 @@ func (s *State) RunStageOp(stage string) func(context.Context) error {
 			}
 
 			internalUtils.Log.Info().Msg("Running rootfs stage")
-			output, err := internalUtils.RunStage("rootfs")
+			output, _ := internalUtils.RunStage("rootfs")
+			internalUtils.Log.Debug().Msg(output.String())
+			err := internalUtils.CreateIfNotExists(constants.LogDir)
+			if err != nil {
+				return err
+			}
 			e := os.WriteFile(filepath.Join(constants.LogDir, "rootfs_stage.log"), output.Bytes(), os.ModePerm)
 			if e != nil {
 				internalUtils.Log.Err(e).Msg("Writing log for rootfs stage")
@@ -96,7 +101,12 @@ func (s *State) RunStageOp(stage string) func(context.Context) error {
 			internalUtils.Log.Info().Msg("Running initramfs stage")
 			chroot := internalUtils.NewChroot(s.Rootdir)
 			return chroot.RunCallback(func() error {
-				output, err := internalUtils.RunStage("initramfs")
+				output, _ := internalUtils.RunStage("initramfs")
+				internalUtils.Log.Debug().Msg(output.String())
+				err := internalUtils.CreateIfNotExists(constants.LogDir)
+				if err != nil {
+					return err
+				}
 				e := os.WriteFile(filepath.Join(constants.LogDir, "initramfs_stage.log"), output.Bytes(), os.ModePerm)
 				if e != nil {
 					internalUtils.Log.Err(e).Msg("Writing log for initramfs stage")