Tag 0.11.6

Create a util sub cmd to add all utils that are handy for development
and already present in the luet codebase. We expose in this case `luet
util unpack` to unpack a docker image without a docker daemon running.

cmd/install.go

@@ -89,6 +89,7 @@ To force install a package:
 		onlydeps := LuetCfg.Viper.GetBool("onlydeps")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
 		yes := LuetCfg.Viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
 		dbpath := LuetCfg.Viper.GetString("system.database_path")
 		rootfs := LuetCfg.Viper.GetString("system.rootfs")
@@ -121,6 +122,7 @@ To force install a package:
 			Force:                       force,
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
+			DownloadOnly:                downloadOnly,
 			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
@@ -147,6 +149,7 @@ func init() {
 	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
 	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	installCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(installCmd)
 }

cmd/replace.go

@@ -66,6 +66,7 @@ var replaceCmd = &cobra.Command{
 		dbpath := LuetCfg.Viper.GetString("system.database_path")
 		rootfs := LuetCfg.Viper.GetString("system.rootfs")
 		engine := LuetCfg.Viper.GetString("system.database_engine")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
 		LuetCfg.System.DatabaseEngine = engine
 		LuetCfg.System.DatabasePath = dbpath
@@ -121,6 +122,7 @@ var replaceCmd = &cobra.Command{
 			OnlyDeps:                    onlydeps,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
 		})
 		inst.Repositories(repos)
 
@@ -148,6 +150,7 @@ func init() {
 	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
 	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+	replaceCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(replaceCmd)
 }

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.11.3"
+	LuetCLIVersion = "0.11.6"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/upgrade.go

@@ -66,6 +66,7 @@ var upgradeCmd = &cobra.Command{
 		dbpath := LuetCfg.Viper.GetString("system.database_path")
 		rootfs := LuetCfg.Viper.GetString("system.rootfs")
 		engine := LuetCfg.Viper.GetString("system.database_engine")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
 
 		LuetCfg.System.DatabaseEngine = engine
 		LuetCfg.System.DatabasePath = dbpath
@@ -96,6 +97,7 @@ var upgradeCmd = &cobra.Command{
 			UpgradeNewRevisions:         sync,
 			PreserveSystemEssentialData: true,
 			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
 		})
 		inst.Repositories(repos)
 
@@ -123,6 +125,7 @@ func init() {
 	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
 	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	upgradeCmd.Flags().Bool("download-only", false, "Download only")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

cmd/util.go

@@ -0,0 +1,36 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+)
+
+var utilGroup = &cobra.Command{
+	Use:   "util [command] [OPTIONS]",
+	Short: "General luet internal utilities exposed",
+}
+
+func init() {
+	RootCmd.AddCommand(utilGroup)
+
+	utilGroup.AddCommand(
+		NewUnpackCommand(),
+	)
+}

cmd/util/unpack.go

@@ -0,0 +1,72 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/spf13/cobra"
+)
+
+func NewUnpackCommand() *cobra.Command {
+
+	return &cobra.Command{
+		Use:   "unpack image path",
+		Short: "Unpack a docker image natively",
+		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
+		
+	luet util unpack golang:alpine /alpine
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+
+			if len(args) != 2 {
+				Fatal("Expects an image and a path")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			destination, err := filepath.Abs(args[1])
+			if err != nil {
+				Error("Invalid path %s", destination)
+				os.Exit(1)
+			}
+
+			temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+			if err != nil {
+				Fatal("Cannot create a tempdir", err.Error())
+			}
+
+			Info("Downloading", image, "to", destination)
+			info, err := helpers.DownloadAndExtractDockerImage(temp, image, destination)
+			if err != nil {
+				Error(err.Error())
+				os.Exit(1)
+			}
+			Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+		},
+	}
+
+}

pkg/helpers/docker.go

@@ -0,0 +1,44 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"os"
+
+	"github.com/mudler/luet/pkg/helpers/imgworker"
+	"github.com/pkg/errors"
+)
+
+// DownloadAndExtractDockerImage is a re-adaption
+// from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
+func DownloadAndExtractDockerImage(temp, image, dest string) (*imgworker.ListedImage, error) {
+	defer os.RemoveAll(temp)
+	c, err := imgworker.New(temp)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed creating client")
+	}
+	defer c.Close()
+
+	listedImage, err := c.Pull(image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed listing images")
+
+	}
+
+	os.RemoveAll(dest)
+	err = c.Unpack(image, dest)
+	return listedImage, err
+}

pkg/installer/client/docker.go

@@ -24,8 +24,6 @@ import (
 	"github.com/docker/go-units"
 	"github.com/pkg/errors"
 
-	imgworker "github.com/mudler/luet/pkg/installer/client/imgworker"
-
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -40,33 +38,6 @@ func NewDockerClient(r RepoData) *DockerClient {
 	return &DockerClient{RepoData: r}
 }
 
-func downloadAndExtractDockerImage(image, dest string) error {
-	temp, err := config.LuetCfg.GetSystem().TempDir("contentstore")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(temp)
-	Debug("Temporary directory", temp)
-	c, err := imgworker.New(temp)
-	if err != nil {
-		return errors.Wrapf(err, "failed creating client")
-	}
-	defer c.Close()
-
-	// FROM Slightly adapted from genuinetools/img https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go
-	Debug("Pulling image", image)
-	listedImage, err := c.Pull(image)
-	if err != nil {
-		return errors.Wrapf(err, "failed listing images")
-
-	}
-	Debug("Pulled:", listedImage.Target.Digest)
-	Debug("Size:", units.BytesSize(float64(listedImage.ContentSize)))
-	Debug("Unpacking", image, "to", dest)
-	os.RemoveAll(dest)
-	return c.Unpack(image, dest)
-}
-
 func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
 	//var u *url.URL = nil
 	var err error
@@ -107,15 +78,24 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
 
 		for _, uri := range c.RepoData.Urls {
 
-			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+			imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().ImageID())
 			Info("Downloading image", imageName)
 
+			contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+			if err != nil {
+				Warning("Cannot create contentstore", err.Error())
+				continue
+			}
+
 			// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
-			err = downloadAndExtractDockerImage(imageName, temp)
+			info, err := helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp)
 			if err != nil {
 				Debug("Failed download of image", imageName)
 				continue
 			}
+
+			Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+			Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
 			Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
 
 			newart := artifact
@@ -162,16 +142,24 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
 			continue
 		}
 
-		Debug("Downloading file", name, "from", uri)
+		contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore")
+		if err != nil {
+			Warning("Cannot create contentstore", err.Error())
+			continue
+		}
 
 		imageName := fmt.Sprintf("%s:%s", uri, name)
-		//imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name)
-		err = downloadAndExtractDockerImage(imageName, temp)
+		Info("Downloading", imageName)
+
+		info, err := helpers.DownloadAndExtractDockerImage(contentstore, imageName, temp)
 		if err != nil {
 			Debug("Failed download of image", imageName)
 			continue
 		}
 
+		Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
+		Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
+
 		Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
 		err = helpers.CopyFile(filepath.Join(temp, name), file.Name())
 

pkg/installer/installer.go

@@ -47,6 +47,7 @@ type LuetInstallerOptions struct {
 	CheckConflicts                                                 bool
 	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
 	Ask                                                            bool
+	DownloadOnly                                                   bool
 }
 
 type LuetInstaller struct {
@@ -304,6 +305,9 @@ func (l *LuetInstaller) swap(syncedRepos Repositories, toRemove pkg.Packages, to
 	if err := l.download(syncedRepos, match); err != nil {
 		return errors.Wrap(err, "Pre-downloading packages")
 	}
+	if l.Options.DownloadOnly {
+		return nil
+	}
 
 	err = l.Uninstall(s, toRemove...)
 	if err != nil && !l.Options.Force {
@@ -530,6 +534,10 @@ func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]A
 		return errors.Wrap(err, "Downloading packages")
 	}
 
+	if l.Options.DownloadOnly {
+		return nil
+	}
+
 	all := make(chan ArtifactMatch)
 
 	wg := new(sync.WaitGroup)

pkg/installer/repository.go

@@ -333,7 +333,7 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string,
 			return nil
 		}
 
-		packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().GetFingerPrint())
+		packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID())
 
 		if imagePush && b.ImageAvailable(packageImage) && !force {
 			Info("Image", packageImage, "already present, skipping. use --force-push to override")

pkg/package/package.go

@@ -46,7 +46,7 @@ type Package interface {
 
 	GetFingerPrint() string
 	GetPackageName() string
-	GetPackageImageName() string
+	ImageID() string
 	Requires([]*DefaultPackage) Package
 	Conflicts([]*DefaultPackage) Package
 	Revdeps(PackageDatabase) Packages
@@ -289,8 +289,8 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
-func (p *DefaultPackage) GetPackageImageName() string {
-	return fmt.Sprintf("%s-%s:%s", p.Name, p.Category, p.Version)
+func (p *DefaultPackage) ImageID() string {
+	return strings.ReplaceAll(p.GetFingerPrint(), "+", "-")
 }
 
 // GetBuildTimestamp returns the package build timestamp

pkg/package/package_test.go

@@ -66,6 +66,17 @@ var _ = Describe("Package", func() {
 		})
 	})
 
+	Context("ImageID", func() {
+		It("Returns a correct ImageID escaping unsupported chars", func() {
+			p := NewPackage("A", "1.0+p1", []*DefaultPackage{}, []*DefaultPackage{})
+			Expect(p.ImageID()).To(Equal("A--1.0-p1"))
+		})
+		It("Returns a correct ImageID", func() {
+			p := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})
+			Expect(p.ImageID()).To(Equal("A--1.0"))
+		})
+	})
+
 	Context("Find label on packages", func() {
 		a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
 		a.AddLabel("project1", "test1")

tests/integration/01_simple_downloadonly.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_engine: "memory"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testDownloadOnly() {
+    luet install -y --download-only --config $tmpdir/luet.yaml test/c > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package not installed' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'cache populated' "[ -e '$tmpdir/testrootfs/var/cache/luet/packages/c-test-1.0.package.tar.gz' ]"
+
+    luet install -y --config $tmpdir/luet.yaml test/c > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+