Re-introduce local runner (#57)

* Re-introduce local runner

Signed-off-by: Mauro Morales <mauro.morales@spectrocloud.com>

* Refactor code, remove LocalRunner

and allow both "container:" and "run:" types to use local archives

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

* Treat "docker:" bundler prefix the same as "container:"

as per the docs:
https://kairos.io/docs/advanced/bundles/#bundle-types

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

* Add TODO tests

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

* Remove the not-needed LocalRunner

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

* Implement tests

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

* Remove TODO

Won't do now to avoid introducing bugs

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

---------

Signed-off-by: Mauro Morales <mauro.morales@spectrocloud.com>
Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>
Co-authored-by: Dimitris Karakasilis <dimitris@karakasilis.me>

bundles/bundle_test.go

@@ -1,7 +1,9 @@
 package bundles_test
 
 import (
+	"io"
 	"os"
+	"path"
 	"path/filepath"
 
 	. "github.com/kairos-io/kairos-sdk/bundles"
@@ -11,7 +13,7 @@ import (
 
 var _ = Describe("Bundle", func() {
 	Context("install", func() {
-		PIt("installs packages from luet repos", func() {
+		It("installs packages from luet repos", func() {
 			dir, err := os.MkdirTemp("", "test")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(dir)
@@ -29,5 +31,177 @@ var _ = Describe("Bundle", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(filepath.Join(dir, "usr", "bin", "edgevpn")).To(BeARegularFile())
 		})
+
+		When("local is true", func() {
+			var installer BundleInstaller
+			var config *BundleConfig
+			var tmpDir, tmpFile string
+			var err error
+
+			BeforeEach(func() {
+				config = &BundleConfig{
+					LocalFile: true,
+				}
+			})
+
+			AfterEach(func() {
+				os.RemoveAll(tmpDir)
+			})
+
+			JustBeforeEach(func() {
+				installer, err = NewBundleInstaller(*config)
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			When("type is container", func() {
+				BeforeEach(func() {
+					tmpDir, err = os.MkdirTemp("", "test")
+					Expect(err).ToNot(HaveOccurred())
+					tmpFile = path.Join(tmpDir, "grub-config.tar")
+					copyFile("../assets/grub-config.tar", tmpFile)
+
+					config.Target = "container://" + tmpFile
+					config.DBPath = "/usr/local/.kairos/db"
+					config.RootPath = "/"
+				})
+
+				It("installs", func() {
+					expectInstalled(installer, config)
+				})
+			})
+
+			When("type is docker", func() {
+				BeforeEach(func() {
+					tmpDir, err = os.MkdirTemp("", "test")
+					Expect(err).ToNot(HaveOccurred())
+					tmpFile = path.Join(tmpDir, "grub-config.tar")
+					copyFile("../assets/grub-config.tar", tmpFile)
+
+					config.Target = "docker://" + tmpFile
+					config.DBPath = "/usr/local/.kairos/db"
+					config.RootPath = "/"
+				})
+
+				It("installs", func() {
+					expectInstalled(installer, config)
+				})
+			})
+
+			When("type is run", func() {
+				BeforeEach(func() {
+					// Ensure no leftovers from previous tests
+					// These tests are meant to run in a container (Earthly), so it should
+					// be ok to delete files like this.
+					os.RemoveAll("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+
+					tmpDir, err = os.MkdirTemp("", "test")
+					Expect(err).ToNot(HaveOccurred())
+					tmpFile = path.Join(tmpDir, "longhorn-bundle.tar")
+					copyFile("../assets/longhorn-bundle.tar", tmpFile)
+					config.Target = "run://" + tmpFile
+				})
+
+				It("installs", func() {
+					_, err := os.Stat("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+					Expect(err).To(HaveOccurred())
+
+					err = installer.Install(config)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = os.Stat("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+					Expect(err).ToNot(HaveOccurred())
+				})
+			})
+		})
+
+		When("local is false", func() {
+			var installer BundleInstaller
+			var config *BundleConfig
+			var err error
+
+			BeforeEach(func() {
+				config = &BundleConfig{
+					LocalFile: false,
+				}
+			})
+
+			JustBeforeEach(func() {
+				installer, err = NewBundleInstaller(*config)
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			When("type is container", func() {
+				BeforeEach(func() {
+					config.Target = "container://quay.io/kairos/packages:grub-config-static-0.9"
+					config.DBPath = "/usr/local/.kairos/db"
+					config.RootPath = "/"
+				})
+
+				It("installs", func() {
+					expectInstalled(installer, config)
+				})
+			})
+
+			When("type is docker", func() {
+				BeforeEach(func() {
+					config.Target = "docker://quay.io/kairos/packages:grub-config-static-0.9"
+					config.DBPath = "/usr/local/.kairos/db"
+					config.RootPath = "/"
+				})
+
+				It("installs", func() {
+					expectInstalled(installer, config)
+				})
+			})
+
+			When("type is run", func() {
+				BeforeEach(func() {
+					os.RemoveAll("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+					config.Target = "run://quay.io/kairos/community-bundles:longhorn_latest"
+				})
+
+				It("installs", func() {
+					_, err := os.Stat("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+					Expect(err).To(HaveOccurred())
+
+					err = installer.Install(config)
+					Expect(err).ToNot(HaveOccurred())
+					_, err = os.Stat("/var/lib/rancher/k3s/server/manifests/longhorn.yaml")
+					Expect(err).ToNot(HaveOccurred())
+				})
+			})
+		})
 	})
 })
+
+// Copied from: https://opensource.com/article/18/6/copying-files-go
+func copyFile(src, dst string) {
+	sourceFileStat, err := os.Stat(src)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(sourceFileStat.Mode().IsRegular()).To(BeTrue())
+
+	source, err := os.Open(src)
+	Expect(err).ToNot(HaveOccurred())
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	Expect(err).ToNot(HaveOccurred())
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func expectInstalled(installer BundleInstaller, config *BundleConfig) {
+	// Ensure no leftovers from previous tests
+	// These tests are meant to run in a container (Earthly), so it should
+	// be ok to delete files like this.
+	err := os.RemoveAll("/etc/cos/grub.cfg")
+	Expect(err).ToNot(HaveOccurred())
+	_, err = os.Stat("/etc/cos/grub.cfg")
+	Expect(err).To(HaveOccurred())
+
+	err = installer.Install(config)
+	Expect(err).ToNot(HaveOccurred())
+	_, err = os.Stat("/etc/cos/grub.cfg")
+	Expect(err).ToNot(HaveOccurred())
+}

bundles/bundles.go

@@ -1,11 +1,14 @@
 package bundles
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/hashicorp/go-multierror"
 	"github.com/kairos-io/kairos-sdk/utils"
 )
@@ -80,6 +83,22 @@ func (bc *BundleConfig) extractRepo() (string, string, error) {
 	return s[0], s[1], nil
 }
 
+func (bc *BundleConfig) TargetScheme() (string, error) {
+	dat := strings.Split(bc.Target, "://")
+	if len(dat) != 2 {
+		return "", errors.New("invalid target")
+	}
+	return strings.ToLower(dat[0]), nil
+}
+
+func (bc *BundleConfig) TargetNoScheme() (string, error) {
+	dat := strings.Split(bc.Target, "://")
+	if len(dat) != 2 {
+		return "", errors.New("invalid target")
+	}
+	return dat[1], nil
+}
+
 func defaultConfig() *BundleConfig {
 	return &BundleConfig{
 		DBPath:     "/usr/local/.kairos/db",
@@ -113,12 +132,6 @@ func RunBundles(bundles ...[]BundleOption) error {
 			resErr = multierror.Append(err)
 			continue
 		}
-		dat := strings.Split(config.Target, "://")
-		if len(dat) != 2 {
-			resErr = multierror.Append(fmt.Errorf("invalid target"))
-			continue
-		}
-		config.Target = dat[1]
 
 		err = installer.Install(config)
 		if err != nil {
@@ -131,26 +144,31 @@ func RunBundles(bundles ...[]BundleOption) error {
 }
 
 func NewBundleInstaller(bc BundleConfig) (BundleInstaller, error) {
-
+	scheme, err := bc.TargetScheme()
-	dat := strings.Split(bc.Target, "://")
+	if err != nil {
-	if len(dat) != 2 {
+		return nil, err
-		return nil, fmt.Errorf("could not decode scheme")
 	}
-	switch strings.ToLower(dat[0]) {
+
-	case "container":
+	switch scheme {
-		return &OCIImageExtractor{}, nil
+	case "container", "docker":
+		return &OCIImageExtractor{
+			Local: bc.LocalFile,
+		}, nil
 	case "run":
-		return &OCIImageRunner{}, nil
+		return &OCIImageRunner{
+			Local: bc.LocalFile,
+		}, nil
 	case "package":
 		return &LuetInstaller{}, nil
-
 	}
 
 	return &LuetInstaller{}, nil
 }
 
 // OCIImageExtractor will extract an OCI image
-type OCIImageExtractor struct{}
+type OCIImageExtractor struct {
+	Local bool
+}
 
 func (e OCIImageExtractor) Install(config *BundleConfig) error {
 	if !utils.Exists(config.RootPath) {
@@ -159,11 +177,29 @@ func (e OCIImageExtractor) Install(config *BundleConfig) error {
 			return fmt.Errorf("could not create destination path %s: %s", config.RootPath, err)
 		}
 	}
-	return utils.ExtractOCIImage(config.Target, config.RootPath, utils.GetCurrentPlatform())
+
+	var img v1.Image
+	var err error
+	target, err := config.TargetNoScheme()
+	if err != nil {
+		return err
+	}
+	if e.Local {
+		img, err = tarball.ImageFromPath(target, nil)
+	} else {
+		img, err = utils.GetImage(target, utils.GetCurrentPlatform())
+	}
+	if err != nil {
+		return err
+	}
+
+	return utils.ExtractOCIImage(img, config.RootPath)
 }
 
 // OCIImageRunner will extract an OCI image and then run its run.sh
-type OCIImageRunner struct{}
+type OCIImageRunner struct {
+	Local bool
+}
 
 func (e OCIImageRunner) Install(config *BundleConfig) error {
 	tempDir, err := os.MkdirTemp("", "containerrunner")
@@ -172,16 +208,30 @@ func (e OCIImageRunner) Install(config *BundleConfig) error {
 	}
 	defer os.RemoveAll(tempDir)
 
-	err = utils.ExtractOCIImage(config.Target, tempDir, utils.GetCurrentPlatform())
+	var img v1.Image
+	target, err := config.TargetNoScheme()
+	if err != nil {
+		return err
+	}
+	if e.Local {
+		img, err = tarball.ImageFromPath(target, nil)
+	} else {
+		img, err = utils.GetImage(target, utils.GetCurrentPlatform())
+	}
+	if err != nil {
+		return err
+	}
+
+	err = utils.ExtractOCIImage(img, tempDir)
 	if err != nil {
 		return err
 	}
 
 	// We want to expect tempDir as context
 	out, err := utils.SHInDir(
-		filepath.Join(tempDir, "run.sh"),
+		"/bin/sh run.sh",
 		tempDir,
-		fmt.Sprintf("CONTAINERDIR=%s", tempDir), fmt.Sprintf("BUNDLE_TARGET=%s", config.Target))
+		fmt.Sprintf("CONTAINERDIR=%s", tempDir), fmt.Sprintf("BUNDLE_TARGET=%s", target))
 	if err != nil {
 		return fmt.Errorf("could not execute container: %w - %s", err, out)
 	}
@@ -215,12 +265,16 @@ func (l *LuetInstaller) Install(config *BundleConfig) error {
 		return fmt.Errorf("could not add repository: %w - %s", err, out)
 	}
 
+	target, err := config.TargetNoScheme()
+	if err != nil {
+		return err
+	}
 	out, err = utils.SH(
 		fmt.Sprintf(
 			`LUET_CONFIG_FROM_HOST=false luet install -y  --system-dbpath %s --system-target %s %s`,
 			config.DBPath,
 			config.RootPath,
-			config.Target,
+			target,
 		),
 	)
 	if err != nil {

utils/image.go

@@ -4,6 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"strings"
+	"syscall"
+	"time"
+
 	"github.com/containerd/containerd/archive"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/logs"
@@ -13,12 +20,6 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
-	"io"
-	"net/http"
-	"runtime"
-	"strings"
-	"syscall"
-	"time"
 )
 
 var defaultRetryBackoff = remote.Backoff{
@@ -41,27 +42,17 @@ var defaultRetryPredicate = func(err error) bool {
 }
 
 // ExtractOCIImage will extract a given targetImage into a given targetDestination
-func ExtractOCIImage(targetImage, targetDestination, targetPlatform string) error {
+func ExtractOCIImage(img v1.Image, targetDestination string) error {
-	var img v1.Image
-	var err error
-
-	img, err = getimage(targetImage, targetPlatform)
-	if err != nil {
-		return err
-	}
-
 	reader := mutate.Extract(img)
 
-	_, err = archive.Apply(context.Background(), targetDestination, reader)
+	_, err := archive.Apply(context.Background(), targetDestination, reader)
-	if err != nil {
+
-		return err
+	return err
-	}
-	return nil
 }
 
-// image returns the proper image to pull with transport and auth
+// GetImage if returns the proper image to pull with transport and auth
 // tries local daemon first and then fallbacks into remote
-func getimage(targetImage, targetPlatform string) (v1.Image, error) {
+func GetImage(targetImage, targetPlatform string) (v1.Image, error) {
 	var platform *v1.Platform
 	var image v1.Image
 	var err error
@@ -106,7 +97,7 @@ func GetOCIImageSize(targetImage, targetPlatform string) (int64, error) {
 	var img v1.Image
 	var err error
 
-	img, err = getimage(targetImage, targetPlatform)
+	img, err = GetImage(targetImage, targetPlatform)
 	if err != nil {
 		return size, err
 	}

utils/utils.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"image"
 	"net"
 	"os"
@@ -14,6 +13,8 @@ import (
 	"runtime"
 	"strings"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/denisbrodbeck/machineid"
 	"github.com/joho/godotenv"
 	"github.com/pterm/pterm"