🆕 Tag 0.20.9

Drop also direct xattrs handling

cmd/build.go

@@ -110,6 +110,9 @@ Build packages specifying multiple definition trees:
 		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
 		full, _ := cmd.Flags().GetBool("full")
 		rebuild, _ := cmd.Flags().GetBool("rebuild")
+		pushFinalImages, _ := cmd.Flags().GetBool("push-final-images")
+		pushFinalImagesRepository, _ := cmd.Flags().GetString("push-final-images-repository")
+		pushFinalImagesForce, _ := cmd.Flags().GetBool("push-final-images-force")
 
 		var results Results
 		backendArgs := viper.GetStringSlice("backend-args")
@@ -159,8 +162,7 @@ Build packages specifying multiple definition trees:
 
 		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
 
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
-			options.NoDeps(nodeps),
+		compileropts := []options.Option{options.NoDeps(nodeps),
 			options.WithBackendType(backendType),
 			options.PushImages(push),
 			options.WithBuildValues(values),
@@ -177,8 +179,21 @@ Build packages specifying multiple definition trees:
 			options.WithContext(util.DefaultContext),
 			options.BackendArgs(backendArgs),
 			options.Concurrency(concurrency),
-			options.WithCompressionType(compression.Implementation(compressionType)),
-		)
+			options.WithCompressionType(compression.Implementation(compressionType))}
+
+		if pushFinalImages {
+			compileropts = append(compileropts, options.EnablePushFinalImages)
+			if pushFinalImagesForce {
+				compileropts = append(compileropts, options.ForcePushFinalImages)
+			}
+			if pushFinalImagesRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(pushFinalImagesRepository))
+			} else if imageRepository != "" {
+				compileropts = append(compileropts, options.WithFinalRepository(imageRepository))
+			}
+		}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), compileropts...)
 
 		if full {
 			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
@@ -302,6 +317,11 @@ func init() {
 	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+
+	buildCmd.Flags().Bool("push-final-images", false, "Push final images while building")
+	buildCmd.Flags().Bool("push-final-images-force", false, "Override existing images")
+	buildCmd.Flags().String("push-final-images-repository", "", "Repository where to push final images to")
+
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
 	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
 	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.7"
+	LuetCLIVersion = "0.20.9"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/api/core/image/create_test.go

@@ -47,7 +47,7 @@ var _ = Describe("Create", func() {
 			img, err := b.ImageReference("alpine", false)
 			Expect(err).ToNot(HaveOccurred())
 
-			_, dir, err := Extract(ctx, img, false, nil)
+			_, dir, err := Extract(ctx, img, nil)
 			Expect(err).ToNot(HaveOccurred())
 
 			defer os.RemoveAll(dir)
@@ -69,7 +69,7 @@ var _ = Describe("Create", func() {
 			img, err = b.ImageReference("testimage", false)
 			Expect(err).ToNot(HaveOccurred())
 
-			_, dir, err = Extract(ctx, img, false, nil)
+			_, dir, err = Extract(ctx, img, nil)
 			Expect(err).ToNot(HaveOccurred())
 
 			defer os.RemoveAll(dir)

pkg/api/core/image/delta_test.go

@@ -74,7 +74,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -94,7 +93,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -109,7 +107,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -124,7 +121,6 @@ var _ = Describe("Delta", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img2,
-					true,
 					f,
 				)
 				Expect(err).ToNot(HaveOccurred())

pkg/api/core/image/extract.go

@@ -19,14 +19,11 @@ import (
 	"archive/tar"
 	"context"
 	"io"
-	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	containerdarchive "github.com/containerd/containerd/archive"
-	"github.com/docker/docker/pkg/system"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/mudler/luet/pkg/api/core/types"
@@ -196,7 +193,7 @@ func ExtractFiles(
 // ExtractReader perform the extracting action over the io.ReadCloser
 // it extracts the files over output. Accepts a filter as an option
 // and additional containerd Options
-func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	defer reader.Close()
 
 	// If no filter is specified, grab all.
@@ -204,36 +201,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 		filter = func(h *tar.Header) (bool, error) { return true, nil }
 	}
 
-	// Keep records of permissions as we walk the tar
-	type permData struct {
-		PAX, Xattrs map[string]string
-		Uid, Gid    int
-		Name        string
-		FileMode    fs.FileMode
-	}
-
-	permstore, err := ctx.Config.System.TempDir("permstore")
-	if err != nil {
-		return 0, "", err
-	}
-	perms := NewCache(permstore, 50*1024*1024, 10000)
-
-	f := func(h *tar.Header) (bool, error) {
-		res, err := filter(h)
-		if res {
-			perms.SetValue(h.Name, permData{
-				PAX: h.PAXRecords,
-				Uid: h.Uid, Gid: h.Gid,
-				Xattrs:   h.Xattrs,
-				Name:     h.Name,
-				FileMode: h.FileInfo().Mode(),
-			})
-			//perms = append(perms, })
-		}
-		return res, err
-	}
-
-	opts = append(opts, containerdarchive.WithFilter(f))
+	opts = append(opts, containerdarchive.WithFilter(filter))
 
 	// Handle the extraction
 	c, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
@@ -241,46 +209,19 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 		return 0, "", err
 	}
 
-	// Reconstruct permissions
-	if keepPerms {
-		ctx.Debug("Reconstructing permissions")
-		perms.All(func(cr CacheResult) {
-			p := &permData{}
-			cr.Unmarshal(p)
-			ff := filepath.Join(output, p.Name)
-			if _, err := os.Lstat(ff); err == nil {
-				if err := os.Lchown(ff, p.Uid, p.Gid); err != nil {
-					ctx.Warning(err, "failed chowning file")
-				}
-				ctx.Debug("Set", p.Name, p.FileMode)
-				if err := os.Chmod(ff, p.FileMode); err != nil {
-					ctx.Warning(err, "failed chmod file")
-				}
-			}
-			for _, attrs := range []map[string]string{p.Xattrs, p.PAX} {
-				for k, attr := range attrs {
-					if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil {
-						if errors.Is(err, syscall.ENOTSUP) {
-							ctx.Debug("ignored xattr %s in archive", ff)
-						}
-					}
-				}
-			}
-		})
-	}
 	return c, output, nil
 }
 
 // Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
-func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
 	if err != nil {
 		return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
-	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...)
+	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
 }
 
 // ExtractTo is just syntax sugar around ExtractReader
-func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
-	return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...)
+func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
+	return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
 }

pkg/api/core/image/extract_test.go

@@ -58,7 +58,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -72,7 +71,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{}, []string{}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -86,7 +84,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())
@@ -101,7 +98,6 @@ var _ = Describe("Extract", func() {
 				_, tmpdir, err := Extract(
 					ctx,
 					img,
-					true,
 					ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),
 				)
 				Expect(err).ToNot(HaveOccurred())

pkg/api/core/types/artifact/artifact.go

@@ -70,8 +70,8 @@ type PackageArtifact struct {
 	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
 }
 
-func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, keepPerms bool, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
-	_, tmpdiffs, err := image.Extract(ctx, img, keepPerms, filter)
+func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+	_, tmpdiffs, err := image.Extract(ctx, img, filter)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}
@@ -570,7 +570,7 @@ func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool)
 	// 	//	tarModifier.Modifier()
 	// 	return true, nil
 	// },
-	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, ctx.Config.GetGeneral().SameOwner, nil)
+	_, _, err = image.ExtractReader(ctx, replacerArchive, dst, nil)
 	return err
 }
 

pkg/api/core/types/artifact/artifact_test.go

@@ -161,7 +161,6 @@ RUN echo bar > /test2`))
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())
@@ -210,7 +209,6 @@ RUN echo bar > /test2`))
 				ctx,
 				img,
 				result,
-				false,
 				nil,
 			)
 			Expect(err).ToNot(HaveOccurred())

pkg/compiler/compiler.go

@@ -255,7 +255,6 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 	_, rootfs, err := image.Extract(
 		cs.Options.Context,
 		img,
-		keepPermissions,
 		image.ExtractFiles(
 			cs.Options.Context,
 			p.GetPackageDir(),
@@ -338,7 +337,6 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
 		ref2,
 		cs.Options.CompressionType,
 		p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")),
-		keepPermissions,
 		filter,
 	)
 	if err != nil {
@@ -488,12 +486,14 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 
 		a.CompileSpec = p
 		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
-
 		err = a.WriteYAML(p.GetOutputPath())
 		if err != nil {
 			return a, errors.Wrap(err, "Failed while writing metadata file")
 		}
 		cs.Options.Context.Success(pkgTag, "   :white_check_mark: done (empty virtual package)")
+		if cs.Options.PushFinalImages {
+			cs.pushFinalArtifact(a, p, keepPermissions)
+		}
 		return a, nil
 	}
 
@@ -523,11 +523,53 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if err != nil {
 		return a, errors.Wrap(err, "Failed while writing metadata file")
 	}
-	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done")
+	cs.Options.Context.Success(pkgTag, "   :white_check_mark: Done building")
+
+	if cs.Options.PushFinalImages {
+		cs.pushFinalArtifact(a, p, keepPermissions)
+	}
 
 	return a, nil
 }
 
+// TODO: A small readaptation of repository_docker.go pushImageFromArtifact()
+//       Move this to a common place
+func (cs *LuetCompiler) pushFinalArtifact(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
+	cs.Options.Context.Info("Pushing final image for", a.CompileSpec.Package.HumanReadableString())
+	imageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.Package.ImageID())
+
+	// First push the package image
+	if !cs.Backend.ImageAvailable(imageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating and pushing final image for", a.CompileSpec.Package.HumanReadableString(), "as", imageID)
+
+		if err := a.GenerateFinalImage(cs.Options.Context, imageID, cs.GetBackend(), true); err != nil {
+			return errors.Wrap(err, "while creating final image")
+		}
+		if err := cs.Backend.Push(backend.Options{ImageName: imageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", imageID)
+		}
+	}
+
+	// Then the image ID
+	metadataImageID := fmt.Sprintf("%s:%s", cs.Options.PushFinalImagesRepository, a.CompileSpec.GetPackage().GetMetadataFilePath())
+	if !cs.Backend.ImageAvailable(metadataImageID) || cs.Options.PushFinalImagesForce {
+		cs.Options.Context.Info("Generating metadata image for", a.CompileSpec.Package.HumanReadableString(), metadataImageID)
+
+		a := artifact.NewPackageArtifact(filepath.Join(p.GetOutputPath(), a.CompileSpec.GetPackage().GetMetadataFilePath()))
+		metadataArchive, err := artifact.CreateArtifactForFile(cs.Options.Context, a.Path)
+		if err != nil {
+			return errors.Wrap(err, "failed generating checksums for tree")
+		}
+		if err := metadataArchive.GenerateFinalImage(cs.Options.Context, metadataImageID, cs.Backend, keepPermissions); err != nil {
+			return errors.Wrap(err, "Failed generating metadata tree "+metadataImageID)
+		}
+		if err = cs.Backend.Push(backend.Options{ImageName: metadataImageID}); err != nil {
+			return errors.Wrapf(err, "Could not push image: %s", metadataImageID)
+		}
+	}
+	return nil
+}
+
 func (cs *LuetCompiler) waitForImages(images []string) {
 	if cs.Options.PullFirst && cs.Options.Wait {
 		available, _ := oneOfImagesAvailable(images, cs.Backend)

pkg/compiler/compiler_test.go

@@ -16,10 +16,15 @@
 package compiler_test
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
+	helpers "github.com/mudler/luet/tests/helpers"
+
+	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	. "github.com/mudler/luet/pkg/compiler"
@@ -853,6 +858,67 @@ var _ = Describe("Compiler", func() {
 			Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
 			Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
 		})
+
+		It("Pushes final images along", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			randString := strings.ToLower(helpers.String(10))
+			imageName := fmt.Sprintf("ttl.sh/%s", randString)
+			b := sd.NewSimpleDockerBackend(ctx)
+
+			err := generalRecipe.Load("../../tests/fixtures/packagelayers")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(),
+				options.EnablePushFinalImages, options.ForcePushFinalImages, options.WithFinalRepository(imageName))
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "build", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+			//Expect(len(artifacts[0].Dependencies)).To(Equal(1))
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.ImageID()))).To(BeTrue())
+			Expect(b.ImageAvailable(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()))).To(BeTrue())
+
+			img, err := b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[0].Runtime.ImageID()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err := image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			Expect(fileHelper.Exists(filepath.Join(path, "bin/busybox"))).To(BeTrue())
+
+			img, err = b.ImageReference(fmt.Sprintf("%s:%s", imageName, artifacts[1].Runtime.GetMetadataFilePath()), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, path, err = image.Extract(ctx, img, nil)
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(path) // clean up
+
+			meta := filepath.Join(path, artifacts[1].Runtime.GetMetadataFilePath())
+			Expect(fileHelper.Exists(meta)).To(BeTrue())
+
+			d, err := ioutil.ReadFile(meta)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(d)).To(ContainSubstring(artifacts[1].CompileSpec.GetPackage().GetName()))
+		})
 	})
 
 	Context("Packages which conents are a package folder", func() {

pkg/compiler/types/options/compiler_options.go

@@ -47,6 +47,12 @@ type Compiler struct {
 	// TemplatesFolder. should default to tree/templates
 	TemplatesFolder []string
 
+	// Tells wether to push final container images after building
+	PushFinalImages      bool
+	PushFinalImagesForce bool
+	// Image repository to push to
+	PushFinalImagesRepository string
+
 	Context *types.Context
 }
 
@@ -85,6 +91,25 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error {
 	}
 }
 
+// WithFinalRepository Sets the final repository where to push
+// images of built artifacts
+func WithFinalRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PushFinalImagesRepository = r
+		return nil
+	}
+}
+
+func EnablePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImages = true
+	return nil
+}
+
+func ForcePushFinalImages(cfg *Compiler) error {
+	cfg.PushFinalImagesForce = true
+	return nil
+}
+
 func WithBackendType(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		cfg.BackendType = r
@@ -113,6 +138,8 @@ func WithPullRepositories(r []string) func(cfg *Compiler) error {
 	}
 }
 
+// WithPushRepository Sets the image reference where to push
+// cache images
 func WithPushRepository(r string) func(cfg *Compiler) error {
 	return func(cfg *Compiler) error {
 		if len(cfg.PullImageRepository) == 0 {

pkg/helpers/docker/docker.go

@@ -174,7 +174,6 @@ func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, a
 		ctx,
 		img,
 		dest,
-		true,
 		nil,
 	)
 	if err != nil {

pkg/installer/repository_docker.go

@@ -204,7 +204,6 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 			d.context,
 			img,
 			repoTemp,
-			d.context.Config.GetGeneral().SameOwner,
 			nil,
 		)
 		if err != nil {