diff --git a/internal/constants/constants.go b/internal/constants/constants.go index fafce75..b988873 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -31,6 +31,8 @@ const ( OpUkiInit = "uki-init" OpSentinel = "create-sentinel" OpUkiUdev = "uki-udev" + OpUkiBaseMounts = "uki-base-mounts" + OpUkiKernelModules = "uki-kernel-modules" OpWaitForSysroot = "wait-for-sysroot" PersistentStateTarget = "/usr/local/.state" LogDir = "/run/immucore" diff --git a/internal/utils/cloudinit.go b/internal/utils/cloudinit.go index 9d5ec71..8c6b225 100644 --- a/internal/utils/cloudinit.go +++ b/internal/utils/cloudinit.go @@ -56,9 +56,9 @@ func RunStage(stage string) (bytes.Buffer, error) { var buffer bytes.Buffer log := logrus.New() log.SetOutput(&buffer) - log.SetLevel(logrus.DebugLevel) + log.SetLevel(logrus.InfoLevel) yip := NewYipExecutor(log) - c := console.NewStandardConsole(console.WithLogger(log)) + c := ImmucoreConsole{} stageBefore := fmt.Sprintf("%s.before", stage) stageAfter := fmt.Sprintf("%s.after", stage) diff --git a/internal/utils/cloudinit_console.go b/internal/utils/cloudinit_console.go new file mode 100644 index 0000000..f9c1d41 --- /dev/null +++ b/internal/utils/cloudinit_console.go @@ -0,0 +1,48 @@ +package utils + +import ( + "fmt" + "os/exec" + + "github.com/hashicorp/go-multierror" +) + +// ImmucoreConsole is the console for yip. As we have to hijack the Run method to be able to run under UKI +// To add the paths, we need to create our own console. +type ImmucoreConsole struct { +} + +func (s ImmucoreConsole) Run(cmd string, opts ...func(cmd *exec.Cmd)) (string, error) { + c := PrepareCommandWithPath(cmd) + for _, o := range opts { + o(c) + } + out, err := c.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("failed to run %s: %v", cmd, err) + } + + return string(out), err +} + +func (s ImmucoreConsole) Start(cmd *exec.Cmd, opts ...func(cmd *exec.Cmd)) error { + for _, o := range opts { + o(cmd) + } + return cmd.Run() +} + +func (s ImmucoreConsole) RunTemplate(st []string, template string) error { + var errs error + + for _, svc := range st { + out, err := s.Run(fmt.Sprintf(template, svc)) + if err != nil { + Log.Err(err).Msg("Run template") + Log.Debug().Str("output", out).Msg("Run template") + errs = multierror.Append(errs, err) + continue + } + } + return errs +} diff --git a/internal/utils/common.go b/internal/utils/common.go index 2c026d2..97242b8 100644 --- a/internal/utils/common.go +++ b/internal/utils/common.go @@ -191,6 +191,23 @@ func CommandWithPath(c string) (string, error) { return string(o), err } +// PrepareCommandWithPath prepares a cmd with the proper env +// For running under yip. +func PrepareCommandWithPath(c string) *exec.Cmd { + cmd := exec.Command("/bin/sh", "-c", c) + cmd.Env = os.Environ() + pathAppend := "/usr/bin:/usr/sbin:/bin:/sbin" + // try to extract any existing path from the environment + for _, env := range cmd.Env { + splitted := strings.Split(env, "=") + if splitted[0] == "PATH" { + pathAppend = fmt.Sprintf("%s:%s", pathAppend, splitted[1]) + } + } + cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", pathAppend)) + return cmd +} + // GetHostProcCmdline returns the path to /proc/cmdline // Mainly used to override the cmdline during testing. func GetHostProcCmdline() string { @@ -211,7 +228,6 @@ func GetPartByLabel(label string, attempts int) (string, error) { } for _, d := range blockDevices.Disks { for _, part := range d.Partitions { - Log.Info().Interface("part", part).Str("label", label).Msg("getpartbylabel") if part.FilesystemLabel == label { return filepath.Join("/dev/", part.Name), nil } diff --git a/pkg/mount/dag_steps.go b/pkg/mount/dag_steps.go index f4f1799..3fd351b 100644 --- a/pkg/mount/dag_steps.go +++ b/pkg/mount/dag_steps.go @@ -330,8 +330,9 @@ func (s *State) WriteFstabDagStep(g *herd.Graph) error { // WriteSentinelDagStep sets the sentinel file to identify the boot mode. // This is used by several things to know in which state they are, for example cloud configs. -func (s *State) WriteSentinelDagStep(g *herd.Graph) error { +func (s *State) WriteSentinelDagStep(g *herd.Graph, deps ...string) error { return g.Add(cnst.OpSentinel, + herd.WithDeps(deps...), herd.WithCallback(func(ctx context.Context) error { var sentinel string @@ -376,13 +377,75 @@ func (s *State) WriteSentinelDagStep(g *herd.Graph) error { })) } +func (s *State) UKIMountBaseSystem(g *herd.Graph) error { + type mount struct { + where string + what string + fs string + flags uintptr + data string + } + + return g.Add( + cnst.OpUkiBaseMounts, + herd.WithCallback( + func(ctx context.Context) error { + var err error + mounts := []mount{ + { + "/run", + "tmpfs", + "tmpfs", + syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_RELATIME, + "mode=755", + }, + { + "/sys", + "sysfs", + "sysfs", + syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_RELATIME, + "", + }, + { + "/dev", + "devtmpfs", + "devtmpfs", + syscall.MS_NOSUID, + "mode=755", + }, + { + "/tmp", + "tmpfs", + "tmpfs", + syscall.MS_NOSUID | syscall.MS_NODEV, + "", + }, + } + for _, m := range mounts { + e := os.MkdirAll(m.where, 0755) + if e != nil { + err = multierror.Append(err, e) + internalUtils.Log.Err(e).Msg("Creating dir") + } + e = syscall.Mount(m.what, m.where, m.fs, m.flags, m.data) + if e != nil { + err = multierror.Append(err, e) + internalUtils.Log.Err(e).Str("what", m.what).Str("where", m.where).Str("type", m.fs).Msg("Mounting") + } + } + return err + }, + ), + ) +} + // UKIBootInitDagStep tries to launch /sbin/init in root and pass over the system // booting to the real init process // Drops to emergency if not able to. Panic if it cant even launch emergency. -func (s *State) UKIBootInitDagStep(g *herd.Graph, deps ...string) error { +func (s *State) UKIBootInitDagStep(g *herd.Graph) error { return g.Add(cnst.OpUkiInit, - herd.WithDeps(deps...), - herd.WeakDeps, + herd.WithDeps(), + herd.WithWeakDeps(cnst.OpRemountRootRO, cnst.OpRootfsHook, cnst.OpInitramfsHook, cnst.OpWriteFstab), herd.WithCallback(func(ctx context.Context) error { // Print dag before exit, otherwise its never printed as we never exit the program internalUtils.Log.Info().Msg(s.WriteDAG(g)) @@ -400,11 +463,20 @@ func (s *State) UKIBootInitDagStep(g *herd.Graph, deps ...string) error { } // UKIRemountRootRODagStep remount root read only. -func (s *State) UKIRemountRootRODagStep(g *herd.Graph, deps ...string) error { +func (s *State) UKIRemountRootRODagStep(g *herd.Graph) error { return g.Add(cnst.OpRemountRootRO, - herd.WithDeps(deps...), + herd.WithDeps(cnst.OpRootfsHook), herd.WithCallback(func(ctx context.Context) error { - return syscall.Mount("/", "/", "rootfs", syscall.MS_REMOUNT|syscall.MS_RDONLY, "") + var err error + for i := 1; i < 5; i++ { + time.Sleep(1 * time.Second) + // Should we try to stop udev here? + err = syscall.Mount("", "/", "", syscall.MS_REMOUNT|syscall.MS_RDONLY, "") + if err != nil { + continue + } + } + return err }), ) } @@ -413,6 +485,7 @@ func (s *State) UKIRemountRootRODagStep(g *herd.Graph, deps ...string) error { // Needed if we expect to find devices by label... func (s *State) UKIUdevDaemon(g *herd.Graph) error { return g.Add(cnst.OpUkiUdev, + herd.WithDeps(cnst.OpUkiBaseMounts, cnst.OpUkiKernelModules), herd.WithCallback(func(ctx context.Context) error { // Should probably figure out other udevd binaries.... var udevBin string @@ -448,7 +521,8 @@ func (s *State) UKIUdevDaemon(g *herd.Graph) error { // Mainly block devices and net devices // probably others down the line. func (s *State) LoadKernelModules(g *herd.Graph) error { - return g.Add("kernel-modules", + return g.Add(cnst.OpUkiKernelModules, + herd.WithDeps(cnst.OpUkiBaseMounts), herd.WithCallback(func(ctx context.Context) error { drivers, err := kdetect.ProbeKernelModules("") if err != nil { diff --git a/pkg/mount/dag_uki_boot.go b/pkg/mount/dag_uki_boot.go index ee04db4..46e6ddc 100644 --- a/pkg/mount/dag_uki_boot.go +++ b/pkg/mount/dag_uki_boot.go @@ -7,27 +7,31 @@ import ( // RegisterUKI registers the dag for booting from UKI. func (s *State) RegisterUKI(g *herd.Graph) error { - // Write sentinel - s.LogIfError(s.WriteSentinelDagStep(g), "sentinel") + // Mount basic mounts + s.LogIfError(s.UKIMountBaseSystem(g), "mounting base mounts") + // Write sentinel + s.LogIfError(s.WriteSentinelDagStep(g, cnst.OpUkiBaseMounts), "sentinel") + + // Load needed kernel modules s.LogIfError(s.LoadKernelModules(g), "kernel modules") // Udev for devices discovery s.LogIfError(s.UKIUdevDaemon(g), "udev") // Run rootfs stage - s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel), "uki rootfs") + s.LogIfError(s.RootfsStageDagStep(g, cnst.OpSentinel, cnst.OpUkiUdev), "uki rootfs") // Remount root RO - s.LogIfError(s.UKIRemountRootRODagStep(g, cnst.OpRootfsHook), "remount root") - - // Mount base overlay under /run/overlay - s.LogIfError(s.MountBaseOverlayDagStep(g), "base overlay") + s.LogIfError(s.UKIRemountRootRODagStep(g), "remount root") // Populate state bind mounts, overlay mounts, custom-mounts from /run/cos/cos-layout.env // Requires stage rootfs to have run, which usually creates the cos-layout.env file s.LogIfError(s.LoadEnvLayoutDagStep(g, cnst.OpRootfsHook), "loading cos-layout.env") + // Mount base overlay under /run/overlay + s.LogIfError(s.MountBaseOverlayDagStep(g), "base overlay") + // Mount custom overlays loaded from the /run/cos/cos-layout.env file s.LogIfError(s.MountCustomOverlayDagStep(g), "custom overlays mount") @@ -47,6 +51,6 @@ func (s *State) RegisterUKI(g *herd.Graph) error { herd.WithCallback(s.WriteFstab(s.path("/etc/fstab")))), "fstab") // Handover to /sbin/init - _ = s.UKIBootInitDagStep(g, cnst.OpRemountRootRO, cnst.OpRootfsHook, cnst.OpInitramfsHook) + _ = s.UKIBootInitDagStep(g) return nil } diff --git a/pkg/mount/state.go b/pkg/mount/state.go index 822d117..b02220e 100644 --- a/pkg/mount/state.go +++ b/pkg/mount/state.go @@ -85,7 +85,12 @@ func (s *State) RunStageOp(stage string) func(context.Context) error { } internalUtils.Log.Info().Msg("Running rootfs stage") - output, err := internalUtils.RunStage("rootfs") + output, _ := internalUtils.RunStage("rootfs") + internalUtils.Log.Debug().Msg(output.String()) + err := internalUtils.CreateIfNotExists(constants.LogDir) + if err != nil { + return err + } e := os.WriteFile(filepath.Join(constants.LogDir, "rootfs_stage.log"), output.Bytes(), os.ModePerm) if e != nil { internalUtils.Log.Err(e).Msg("Writing log for rootfs stage") @@ -96,7 +101,12 @@ func (s *State) RunStageOp(stage string) func(context.Context) error { internalUtils.Log.Info().Msg("Running initramfs stage") chroot := internalUtils.NewChroot(s.Rootdir) return chroot.RunCallback(func() error { - output, err := internalUtils.RunStage("initramfs") + output, _ := internalUtils.RunStage("initramfs") + internalUtils.Log.Debug().Msg(output.String()) + err := internalUtils.CreateIfNotExists(constants.LogDir) + if err != nil { + return err + } e := os.WriteFile(filepath.Join(constants.LogDir, "initramfs_stage.log"), output.Bytes(), os.ModePerm) if e != nil { internalUtils.Log.Err(e).Msg("Writing log for initramfs stage")