Fix hooks under encrypted partitions (#702)

* Fix hooks under encrypted partitions

We had a mess of mounting and unmounting things around when we try to
copy things to persistent.

Part of the changes (using the by-label to mount the persistent) are due
to the change in kcrypt. As we set the same label to the encrypted fs
and unencrypted fs, our utils.Mount could get mistaken and return the
first hit, which usually its the encrypted one, and we cannot mount that
one.

This patch brings it up to date.

 - Makes bundles and logs hooks work when we have encrypted persistent.
   It didnt work before.
 - Makes both workflows the same.
 - Locks everything once its over, to not leave encrypted parts around
 - Mounts OEM so kcrypt can read the config if we are using a remote
   server for encryption
 - Mounts by label so there is not a change of getting the wrong device
 - Uses the mount syscall directly. The util can mistake and return the
   actual encrypted part if they both have the same label and finds it
   first

---------

Signed-off-by: Itxaka <itxaka@kairos.io>

.github/workflows/webui.yaml

@@ -17,9 +17,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install earthly
-        uses: Luet-lab/luet-install-action@v1
+        uses: earthly/actions-setup@v1
         with:
-          repository: quay.io/kairos/packages
+          github-token: ${{ secrets.GITHUB_TOKEN }}
-          packages: utils/earthly
       - name: WebUI tests
         run: earthly +webui-tests --GO_VERSION=${{ matrix.go-version }}

go.mod

@@ -12,11 +12,11 @@ require (
 	github.com/erikgeiser/promptkit v0.9.0
 	github.com/google/go-containerregistry v0.20.3
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/jaypipes/ghw v0.13.0 // indirect
+	github.com/jaypipes/ghw v0.15.0 // indirect
 	github.com/joho/godotenv v1.5.1
 	github.com/kairos-io/go-nodepair v0.3.0
 	github.com/kairos-io/kairos-sdk v0.7.3
-	github.com/kairos-io/kcrypt v0.12.2
+	github.com/kairos-io/kcrypt v0.14.0
 	github.com/labstack/echo/v4 v4.13.3
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5
@@ -152,7 +152,8 @@ require (
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/otiai10/copy v1.14.0 // indirect
+	github.com/otiai10/copy v1.14.1 // indirect
+	github.com/otiai10/mint v1.6.3 // indirect
 	github.com/packethost/packngo v0.29.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/phayes/permbits v0.0.0-20190612203442-39d7c581d2ee // indirect

go.sum

@@ -287,6 +287,8 @@ github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/my
 github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
 github.com/jaypipes/ghw v0.13.0 h1:log8MXuB8hzTNnSktqpXMHc0c/2k/WgjOMSUtnI1RV4=
 github.com/jaypipes/ghw v0.13.0/go.mod h1:In8SsaDqlb1oTyrbmTC14uy+fbBMvp+xdqX51MidlD8=
+github.com/jaypipes/ghw v0.15.0 h1:kjn+8fWVtB/DKfwMwpojLFMM6a3zdBF1OnBhAbvJ1BI=
+github.com/jaypipes/ghw v0.15.0/go.mod h1:In8SsaDqlb1oTyrbmTC14uy+fbBMvp+xdqX51MidlD8=
 github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic=
 github.com/jaypipes/pcidb v1.0.1/go.mod h1:6xYUz/yYEyOkIkUt2t2J2folIuZ4Yg6uByCGFXMCeE4=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -304,6 +306,12 @@ github.com/kairos-io/kairos-sdk v0.7.3 h1:OyDSEQVtc1MnRrP3M8d+wref0RA3eZof/FTL5E
 github.com/kairos-io/kairos-sdk v0.7.3/go.mod h1:ZSxP3VgOE2+f/3IdPNcGK7qcYLWl44zV+gq0m+9ovoo=
 github.com/kairos-io/kcrypt v0.12.2 h1:+lr8FGS0AW6D5dWSmaR3+AobL1TBTnOFgCSYctKY+5I=
 github.com/kairos-io/kcrypt v0.12.2/go.mod h1:7SPiHzNMYl4MlxeB30s1YlHDYByTusu7u1mU5Nvicm0=
+github.com/kairos-io/kcrypt v0.13.1-0.20250312192315-531e4ff400a8 h1:od7aDyHW0V+92NmnejUW6OFwYJVkh+NPGmei6nxUpG4=
+github.com/kairos-io/kcrypt v0.13.1-0.20250312192315-531e4ff400a8/go.mod h1:/mu25Htq2tZijQ+Md4oZu09dhaHP3JKI2gvzQWA8cYA=
+github.com/kairos-io/kcrypt v0.13.1-0.20250312202502-cc57c8207fe4 h1:6YNF7d8OpwCwlF5xakBURRQYLSzF6fkNWeHvPr8uElU=
+github.com/kairos-io/kcrypt v0.13.1-0.20250312202502-cc57c8207fe4/go.mod h1:/mu25Htq2tZijQ+Md4oZu09dhaHP3JKI2gvzQWA8cYA=
+github.com/kairos-io/kcrypt v0.14.0 h1:7dEg/gLDJkT06XutnLFwjp8ExfxtgYImF8OaCoVRbpY=
+github.com/kairos-io/kcrypt v0.14.0/go.mod h1:qz/QBCg7phprxHogsjouhlish2Jz92JsmFbJVg2M5cI=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kbinani/screenshot v0.0.0-20230812210009-b87d31814237 h1:YOp8St+CM/AQ9Vp4XYm4272E77MptJDHkwypQHIRl9Q=
@@ -428,8 +436,12 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8=
+github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I=
 github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
 github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
+github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
 github.com/packethost/packngo v0.29.0 h1:gRIhciVZQ/zLNrIdIdbOUyB/Tw5IgoaXyhP4bvE+D2s=
 github.com/packethost/packngo v0.29.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=

internal/agent/hooks/bundles.go

@@ -1,14 +1,20 @@
 package hook
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"syscall"
 
 	config "github.com/kairos-io/kairos-agent/v2/pkg/config"
+	"github.com/kairos-io/kairos-agent/v2/pkg/constants"
 	v1 "github.com/kairos-io/kairos-agent/v2/pkg/types/v1"
 	"github.com/kairos-io/kairos-sdk/bundles"
 	"github.com/kairos-io/kairos-sdk/machine"
+	"github.com/kairos-io/kairos-sdk/utils"
+	kcrypt "github.com/kairos-io/kcrypt/pkg/lib"
 )
 
 // BundlePostInstall install bundles just after installation
@@ -28,13 +34,53 @@ func (b BundlePostInstall) Run(c config.Config, _ v1.Spec) error {
 	// Note that the binding of /usr/local/.state/var-lib-extensions.bind to /var/lib/extensions on active/passive its done by inmmucore based on the
 	// 00_rootfs.yaml config which sets the bind and ephemeral paths.
 	c.Logger.Logger.Debug().Msg("Running BundlePostInstall hook")
+	_ = machine.Umount(constants.PersistentDir)
 
-	machine.Mount("COS_PERSISTENT", "/usr/local") //nolint:errcheck
+	_ = machine.Mount(constants.OEMLabel, constants.OEMPath)
 	defer func() {
-		machine.Umount("/usr/local") //nolint:errcheck
+		_ = machine.Umount(constants.OEMPath)
 	}()
 
-	err := os.MkdirAll("/usr/local/.state/var-lib-extensions.bind", os.ModeDir|os.ModePerm)
+	// Path if we have encrypted persistent and we are not on UKI
+	if len(c.Install.Encrypt) != 0 {
+		err := kcrypt.UnlockAll(false)
+		if err != nil {
+			return err
+		}
+		// Close all the unencrypted partitions at the end!
+		defer func() {
+			for _, p := range c.Install.Encrypt {
+				_, _ = utils.SH("udevadm trigger --type=all || udevadm trigger")
+				syscall.Sync()
+				c.Logger.Debugf("Closing unencrypted /dev/disk/by-label/%s", p)
+				out, err := utils.SH(fmt.Sprintf("cryptsetup close /dev/disk/by-label/%s", p))
+				// There is a known error with cryptsetup that it can't close the device because of a semaphore
+				// doesnt seem to affect anything as the device is closed as expected so we ignore it if it matches the
+				// output of the error
+				if err != nil && !strings.Contains(out, "incorrect semaphore state") {
+					c.Logger.Warnf("could not close /dev/disk/by-label/%s: %s", p, out)
+				}
+			}
+		}()
+	}
+
+	_, _ = utils.SH("udevadm trigger --type=all || udevadm trigger")
+	syscall.Sync()
+	err := c.Syscall.Mount(filepath.Join("/dev/disk/by-label", constants.PersistentLabel), constants.UsrLocalPath, "ext4", 0, "")
+	if err != nil {
+		fmt.Printf("could not mount persistent: %s\n", err)
+		return err
+	}
+
+	defer func() {
+		c.Logger.Debugf("Unmounting persistent partition")
+		err := machine.Umount(constants.UsrLocalPath)
+		if err != nil {
+			c.Logger.Errorf("could not unmount persistent partition: %s", err)
+		}
+	}()
+
+	err = os.MkdirAll("/usr/local/.state/var-lib-extensions.bind", os.ModeDir|os.ModePerm)
 	if c.FailOnBundleErrors && err != nil {
 		return err
 	}
@@ -44,17 +90,12 @@ func (b BundlePostInstall) Run(c config.Config, _ v1.Spec) error {
 	if c.FailOnBundleErrors && err != nil {
 		return err
 	}
-	err = syscall.Mount("/usr/local/.state/var-lib-extensions.bind", "/var/lib/extensions", "", syscall.MS_BIND, "")
+	err = c.Syscall.Mount("/usr/local/.state/var-lib-extensions.bind", "/var/lib/extensions", "", syscall.MS_BIND, "")
 	if c.FailOnBundleErrors && err != nil {
 		return err
 	}
 	defer func() {
-		_ = syscall.Unmount("/var/lib/extensions", 0)
+		_ = machine.Umount("/var/lib/extensions")
-	}()
-
-	machine.Mount("COS_OEM", "/oem") //nolint:errcheck
-	defer func() {
-		machine.Umount("/oem") //nolint:errcheck
 	}()
 
 	opts := c.Install.Bundles.Options()

internal/agent/hooks/kcrypt.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kairos-io/kairos-sdk/machine"
 	kcrypt "github.com/kairos-io/kcrypt/pkg/lib"
 	"path/filepath"
+	"syscall"
 )
 
 type Kcrypt struct{}
@@ -28,11 +29,14 @@ func (k Kcrypt) Run(c config.Config, _ v1.Spec) error {
 	// Config passed during install ends up here, so we need to read it
 	_ = machine.Mount("COS_OEM", "/oem")
 	defer func() {
-		_ = machine.Umount("/oem") //nolint:errcheck
+		err := syscall.Unmount(constants.OEMPath, 0)
+		if err != nil {
+			c.Logger.Errorf("could not unmount Oem partition: %s", err)
+		}
 	}()
 
 	for _, p := range c.Install.Encrypt {
-		_, err := kcrypt.Luksify(p, c.Logger.Logger)
+		_, err := kcrypt.Luksify(p, c.Logger)
 		if err != nil {
 			c.Logger.Errorf("could not encrypt partition: %s", err)
 			if c.FailOnBundleErrors {

internal/agent/hooks/kcrypt_uki.go

@@ -92,7 +92,7 @@ func (k KcryptUKI) Run(c config.Config, spec v1.Spec) error {
 	for _, p := range append([]string{constants.OEMLabel, constants.PersistentLabel}, c.Install.Encrypt...) {
 		c.Logger.Infof("Encrypting %s", p)
 		_ = os.Setenv("SYSTEMD_LOG_LEVEL", "debug")
-		err := kcrypt.LuksifyMeasurements(p, c.BindPublicPCRs, c.BindPCRs, c.Logger.Logger)
+		err := kcrypt.LuksifyMeasurements(p, c.BindPublicPCRs, c.BindPCRs, c.Logger)
 		_ = os.Unsetenv("SYSTEMD_LOG_LEVEL")
 		if err != nil {
 			c.Logger.Errorf("could not encrypt partition: %s", err)
@@ -109,7 +109,7 @@ func (k KcryptUKI) Run(c config.Config, spec v1.Spec) error {
 	_, _ = utils.SH("sync")
 
 	_ = os.Setenv("SYSTEMD_LOG_LEVEL", "debug")
-	err = kcrypt.UnlockAllWithLogger(true, c.Logger.Logger)
+	err = kcrypt.UnlockAllWithLogger(true, c.Logger)
 
 	_ = os.Unsetenv("SYSTEMD_LOG_LEVEL")
 	if err != nil {
@@ -143,7 +143,7 @@ func (k KcryptUKI) Run(c config.Config, spec v1.Spec) error {
 				time.Sleep(time.Duration(i) * time.Second)
 				// Retry the unlock as well, because maybe the partition was not refreshed on time for unlock to unlock it
 				// So no matter how many tries we do, it will still be locked and will never appear
-				err := kcrypt.UnlockAllWithLogger(true, c.Logger.Logger)
+				err := kcrypt.UnlockAllWithLogger(true, c.Logger)
 				if err != nil {
 					c.Logger.Debugf("UnlockAll returned: %s", err)
 				}

internal/agent/hooks/logs.go

@@ -15,25 +15,30 @@ import (
 	"syscall"
 )
 
-// CopyLogs copies all current logs to the persistent partition
+// CopyLogs copies all current logs to the persistent partition.
 // useful during install to keep the livecd logs
 // best effort, no error handling
 type CopyLogs struct{}
 
 func (k CopyLogs) Run(c config.Config, _ v1.Spec) error {
 	c.Logger.Logger.Debug().Msg("Running CopyLogs hook")
-	c.Logger.Debugf("Copying logs to persistent partition")
 	_ = machine.Umount(constants.PersistentDir)
 
+	// Config passed during install ends up here, kcrypt challenger needs to read it if we are using a server for encryption
+	_ = machine.Mount(constants.OEMLabel, constants.OEMPath)
+	defer func() {
+		_ = machine.Umount(constants.OEMPath)
+	}()
+
 	// Path if we have encrypted persistent
 	if len(c.Install.Encrypt) != 0 {
 		err := kcrypt.UnlockAll(false)
 		if err != nil {
 			return err
 		}
-		// Close the unencrypted persistent partition at the end!
+		// Close all the unencrypted partitions at the end!
 		defer func() {
-			for _, p := range []string{constants.PersistentLabel} {
+			for _, p := range c.Install.Encrypt {
 				c.Logger.Debugf("Closing unencrypted /dev/disk/by-label/%s", p)
 				out, err := utils.SH(fmt.Sprintf("cryptsetup close /dev/disk/by-label/%s", p))
 				// There is a known error with cryptsetup that it can't close the device because of a semaphore
@@ -46,12 +51,20 @@ func (k CopyLogs) Run(c config.Config, _ v1.Spec) error {
 		}()
 	}
 
-	err := machine.Mount(constants.PersistentLabel, constants.PersistentDir)
+	_, _ = utils.SH("udevadm trigger --type=all || udevadm trigger")
+	err := c.Syscall.Mount(filepath.Join("/dev/disk/by-label", constants.PersistentLabel), constants.PersistentDir, "ext4", 0, "")
 	if err != nil {
-		c.Logger.Errorf("could not mount persistent partition: %s", err)
+		fmt.Printf("could not mount persistent: %s\n", err)
-		return nil
+		return err
 	}
 
+	defer func() {
+		err := machine.Umount(constants.PersistentDir)
+		if err != nil {
+			c.Logger.Errorf("could not unmount persistent partition: %s", err)
+		}
+	}()
+
 	// Create the directory on persistent
 	varLog := filepath.Join(constants.PersistentDir, ".state", "var-log.bind")
 
@@ -66,11 +79,6 @@ func (k CopyLogs) Run(c config.Config, _ v1.Spec) error {
 		c.Logger.Errorf("could not copy logs to persistent partition: %s", err)
 		return nil
 	}
-	err = machine.Umount(constants.PersistentDir)
-	if err != nil {
-		c.Logger.Errorf("could not unmount persistent partition: %s", err)
-		return nil
-	}
 	syscall.Sync()
 	c.Logger.Debugf("Logs copied to persistent partition")
 	c.Logger.Logger.Debug().Msg("Finish CopyLogs hook")