Tag 0.20.12

When depending on those package otherwise we try to compile the full
tree instead of reconstrucing the image which is result of a join while
keeping the revdep tree invariate

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.9"
+	LuetCLIVersion = "0.20.12"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/util.go

@@ -19,9 +19,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/go-units"
+	"github.com/mudler/luet/pkg/api/core/image"
+	luettypes "github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
 
 	"github.com/mudler/luet/cmd/util"
 	"github.com/mudler/luet/pkg/helpers/docker"
@@ -29,6 +34,55 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func pack(ctx *luettypes.Context, p, dst, imageName, arch, OS string) error {
+
+	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
+	if err != nil {
+		return errors.Wrap(err, "error met while creating tempdir for "+p)
+	}
+	defer os.RemoveAll(tempimage.Name()) // clean up
+
+	if err := image.CreateTar(p, tempimage.Name(), imageName, arch, OS); err != nil {
+		return errors.Wrap(err, "could not create image from tar")
+	}
+
+	return fileHelper.CopyFile(tempimage.Name(), dst)
+}
+
+func NewPackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "pack image src.tar dst.tar",
+		Short: "Pack a standard tar archive as a container image",
+		Long: `Pack creates a tar which can be loaded as an image from a standard flat tar archive, for e.g. with docker load. 
+It doesn't need the docker daemon to run, and allows to override default os/arch:
+		
+	luet util pack --os arm64 image:tag src.tar dst.tar
+`,
+		Args: cobra.MinimumNArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			src := args[1]
+			dst := args[2]
+
+			arch, _ := cmd.Flags().GetString("arch")
+			os, _ := cmd.Flags().GetString("os")
+
+			err := pack(util.DefaultContext, src, dst, image, arch, os)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			util.DefaultContext.Info("Image packed as", image)
+		},
+	}
+
+	c.Flags().String("arch", runtime.GOARCH, "Image architecture")
+	c.Flags().String("os", runtime.GOOS, "Image OS")
+
+	return c
+}
+
 func NewUnpackCommand() *cobra.Command {
 
 	c := &cobra.Command{
@@ -102,5 +156,6 @@ func init() {
 
 	utilGroup.AddCommand(
 		NewUnpackCommand(),
+		NewPackCommand(),
 	)
 }

pkg/api/core/image/create.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"os"
 
+	containerdCompression "github.com/containerd/containerd/archive/compression"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
@@ -27,13 +28,13 @@ import (
 	"github.com/pkg/errors"
 )
 
-func imageFromTar(imagename, architecture, OS string, r io.Reader) (name.Reference, v1.Image, error) {
+func imageFromTar(imagename, architecture, OS string, opener func() (io.ReadCloser, error)) (name.Reference, v1.Image, error) {
 	newRef, err := name.ParseReference(imagename)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	layer, err := tarball.LayerFromReader(r)
+	layer, err := tarball.LayerFromOpener(opener)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -67,28 +68,30 @@ func imageFromTar(imagename, architecture, OS string, r io.Reader) (name.Referen
 
 // CreateTar a imagetarball from a standard tarball
 func CreateTar(srctar, dstimageTar, imagename, architecture, OS string) error {
-	f, err := os.Open(srctar)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+srctar)
-	}
-	defer f.Close()
 
-	return CreateTarReader(f, dstimageTar, imagename, architecture, OS)
-}
-
-// CreateTarReader a imagetarball from a standard tarball
-func CreateTarReader(r io.Reader, dstimageTar, imagename, architecture, OS string) error {
 	dstFile, err := os.Create(dstimageTar)
 	if err != nil {
 		return errors.Wrap(err, "Cannot create "+dstimageTar)
 	}
 	defer dstFile.Close()
 
-	newRef, img, err := imageFromTar(imagename, architecture, OS, r)
+	newRef, img, err := imageFromTar(imagename, architecture, OS, func() (io.ReadCloser, error) {
+		f, err := os.Open(srctar)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+		decompressed, err := containerdCompression.DecompressStream(f)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot open "+srctar)
+		}
+
+		return decompressed, nil
+	})
 	if err != nil {
 		return err
 	}
 
 	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
 	return tarball.Write(newRef, img, dstFile)
+
 }

pkg/api/core/types/artifact/artifact.go

@@ -211,16 +211,6 @@ type ImageBuilder interface {
 
 // GenerateFinalImage takes an artifact and builds a Docker image with its content
 func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) error {
-	archiveFile, err := os.Open(a.Path)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+a.Path)
-	}
-	defer archiveFile.Close()
-
-	decompressed, err := containerdCompression.DecompressStream(archiveFile)
-	if err != nil {
-		return errors.Wrap(err, "Cannot open "+a.Path)
-	}
 
 	tempimage, err := ctx.Config.GetSystem().TempFile("tempimage")
 	if err != nil {
@@ -228,7 +218,7 @@ func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName strin
 	}
 	defer os.RemoveAll(tempimage.Name()) // clean up
 
-	if err := image.CreateTarReader(decompressed, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
+	if err := image.CreateTar(a.Path, tempimage.Name(), imageName, runtime.GOARCH, runtime.GOOS); err != nil {
 		return errors.Wrap(err, "could not create image from tar")
 	}
 

pkg/compiler/compiler.go

@@ -239,19 +239,6 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 		return nil, err
 	}
 
-	// artifact.ImageToArtifact(
-	// 	cs.Options.Context,
-	// 	img,
-	// 	cs.Options.CompressionType,
-	// 	p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"),
-	// 	image.ExtractFiles(
-	// 		cs.Options.Context,
-	// 		strings.TrimLeft(p.GetPackageDir(), "/"),
-	// 		p.GetIncludes(),
-	// 		p.GetExcludes(),
-	// 	),
-	// )
-	// TODO: Trim includes/excludes from "/" ?
 	_, rootfs, err := image.Extract(
 		cs.Options.Context,
 		img,
@@ -492,7 +479,9 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 		}
 		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
 		if cs.Options.PushFinalImages {
-			cs.pushFinalArtifact(a, p, keepPermissions)
+			if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+				return nil, err
+			}
 		}
 		return a, nil
 	}
@@ -526,7 +515,9 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done building")
 
 	if cs.Options.PushFinalImages {
-		cs.pushFinalArtifact(a, p, keepPermissions)
+		if err := cs.pushFinalArtifact(a, p, keepPermissions); err != nil {
+			return nil, err
+		}
 	}
 
 	return a, nil
@@ -551,7 +542,7 @@ func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compil
 	}
 
 	// Then the image ID
-	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.GetPackage().GetMetadataFilePath())
+	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, helpers.SanitizeImageString(a.CompileSpec.GetPackage().GetMetadataFilePath()))
 	if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
 		cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
 
@@ -1086,7 +1077,31 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 
 	ht := NewHashTree(cs.Database)
 
-	packageHashTree, err := ht.Query(cs, p)
+	// When computing the hash tree, we need to take into consideration
+	// that packages that require final images have to be seen as packages without deps
+	// This is because we don't really want to calculate the deptree of them as
+	// as it is handled already when we are creating the images in resolveFinalImages().
+	c := *cs
+	copy := &c
+	memDB := pkg.NewInMemoryDatabase(false)
+	// Create a copy to avoid races
+	dbCopy := pkg.NewInMemoryDatabase(false)
+	err := cs.Database.Clone(dbCopy)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed cloning db")
+	}
+	for _, p := range dbCopy.World() {
+		copy := p.Clone()
+		spec, _ := cs.FromPackage(p)
+		if spec.RequiresFinalImages {
+			copy.Requires([]*pkg.DefaultPackage{})
+		}
+
+		memDB.CreatePackage(copy)
+	}
+	copy.Database = memDB
+
+	packageHashTree, err := ht.Query(copy, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed querying hashtree")
 	}
@@ -1144,6 +1159,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	buildTarget := !cs.Options.OnlyDeps
 
 	if buildDeps {
+
 		cs.Options.Context.Info(":deciduous_tree: Build dependencies for " + p.GetPackage().HumanReadableString())
 		for _, assertion := range dependencies { //highly dependent on the order
 			depsN++
@@ -1159,6 +1175,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 				return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
 			}
 			compileSpec.BuildOptions.PullImageRepository = append(compileSpec.BuildOptions.PullImageRepository, p.BuildOptions.PullImageRepository...)
+
 			cs.Options.Context.Debug("PullImage repos:", compileSpec.BuildOptions.PullImageRepository)
 
 			compileSpec.SetOutputPath(p.GetOutputPath())

pkg/tree/compiler_recipe.go

@@ -174,6 +174,7 @@ func (r *CompilerRecipe) Load(path string) error {
 								filepath.Dir(currentpath))
 					}
 					pack.Requires(packbuild.GetRequires())
+
 					pack.Conflicts(packbuild.GetConflicts())
 				}
 

tests/fixtures/join_complex/c/c1/build.yaml

@@ -7,3 +7,7 @@ requires:
     version: ">=0"
 
 requires_final_images: true
+
+steps:
+- | 
+   /bin/sh -c "if [ -e /usr/bin/generate.sh ]; then ls -liah /usr/bin && exit 1; fi"

tests/fixtures/join_complex/cat/b-1.1/build.yaml

@@ -6,6 +6,7 @@ requires:
 prelude:
   - echo foo > /test
   - echo bar > /test2
+  - cp -rf generate.sh /usr/bin/
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc