Add CreateTar to image API

Add api call which uses go-container registry to create OCI images from
standard tar archives.

Consume new API when generating final images instead of docker building them
and adapts/add tests as necessary.

This change now allows to carry over xattrs to final images.

Fixes #266

pkg/api/core/image/cache.go

@@ -17,7 +17,6 @@ package image
 
 import (
 	"encoding/json"
-	"fmt"
 	"os"
 	"strings"
 
@@ -156,10 +155,7 @@ func (c *Cache) All(fn func(CacheResult)) {
 	}
 
 	for key := range c.store.Keys(nil) {
-		val, err := c.store.Read(key)
-		if err != nil {
-			panic(fmt.Sprintf("key %s had no value", key))
-		}
+		val, _ := c.store.Read(key)
 		fn(CacheResult{key: key, value: string(val)})
 	}
 }

pkg/api/core/image/create.go

@@ -0,0 +1,81 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image
+
+import (
+	"io"
+	"os"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/pkg/errors"
+)
+
+func imageFromTar(imagename string, r io.Reader) (name.Reference, v1.Image, error) {
+	newRef, err := name.ParseReference(imagename)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	layer, err := tarball.LayerFromReader(r)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	img, err := mutate.Append(empty.Image, mutate.Addendum{
+		Layer: layer,
+		History: v1.History{
+			CreatedBy: "luet",
+			Comment:   "Custom image",
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return newRef, img, nil
+}
+
+// CreateTar a imagetarball from a standard tarball
+func CreateTar(srctar, dstimageTar, imagename string) error {
+	f, err := os.Open(srctar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot open "+srctar)
+	}
+	defer f.Close()
+
+	return CreateTarReader(f, dstimageTar, imagename)
+}
+
+// CreateTarReader a imagetarball from a standard tarball
+func CreateTarReader(r io.Reader, dstimageTar, imagename string) error {
+	dstFile, err := os.Create(dstimageTar)
+	if err != nil {
+		return errors.Wrap(err, "Cannot create "+dstimageTar)
+	}
+	defer dstFile.Close()
+
+	newRef, img, err := imageFromTar(imagename, r)
+	if err != nil {
+		return err
+	}
+
+	// NOTE: We might also stream that back to the daemon with daemon.Write(tag, img)
+	return tarball.Write(newRef, img, dstFile)
+}

pkg/api/core/image/create_test.go

@@ -0,0 +1,79 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package image_test
+
+import (
+	"os"
+	"path/filepath"
+
+	. "github.com/mudler/luet/pkg/api/core/image"
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Create", func() {
+	Context("Creates an OCI image from a standard tar", func() {
+		It("creates an image which is loadable", func() {
+			ctx := types.NewContext()
+
+			dst, err := ctx.Config.System.TempFile("dst")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dst.Name())
+			srcTar, err := ctx.Config.System.TempFile("srcTar")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(srcTar.Name())
+
+			b := backend.NewSimpleDockerBackend(ctx)
+
+			b.DownloadImage(backend.Options{ImageName: "alpine"})
+			img, err := b.ImageReference("alpine", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err := Extract(ctx, img, false, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+
+			Expect(file.Touch(filepath.Join(dir, "test"))).ToNot(HaveOccurred())
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+
+			a := artifact.NewPackageArtifact(srcTar.Name())
+			a.Compress(dir, 1)
+
+			// Unfortunately there is no other easy way to test this
+			err = CreateTar(srcTar.Name(), dst.Name(), "testimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			b.LoadImage(dst.Name())
+
+			Expect(b.ImageExists("testimage")).To(BeTrue())
+
+			img, err = b.ImageReference("testimage", false)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, dir, err = Extract(ctx, img, false, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			defer os.RemoveAll(dir)
+			Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue())
+			Expect(file.Exists(filepath.Join(dir, "test"))).To(BeTrue())
+		})
+	})
+})

pkg/api/core/image/extract.go

@@ -32,11 +32,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Extract dir:
-// -> First extract delta considering the dir
-// Afterward create artifact pointing to the dir
-
-// ExtractDeltaFiles returns an handler to extract files in a list
+// ExtractDeltaAdditionsFromImages is a filter that takes two images
+// an includes and an excludes list. It computes the delta between the images
+// considering the added files only, and applies a filter on them based on the regexes
+// in the lists.
 func ExtractDeltaAdditionsFiles(
 	ctx *types.Context,
 	srcimg v1.Image,
@@ -127,6 +126,9 @@ func ExtractDeltaAdditionsFiles(
 	}, nil
 }
 
+// ExtractFiles returns a filter that extracts files from the given path (if not empty)
+// It then filters files by an include and exclude list.
+// The list can be regexes
 func ExtractFiles(
 	ctx *types.Context,
 	prefixPath string,
@@ -190,6 +192,9 @@ func ExtractFiles(
 	}
 }
 
+// ExtractReader perform the extracting action over the io.ReadCloser
+// it extracts the files over output. Accepts a filter as an option
+// and additional containerd Options
 func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	defer reader.Close()
 
@@ -259,6 +264,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep
 	return c, output, nil
 }
 
+// Extract is just syntax sugar around ExtractReader. It extracts an image into a dir
 func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
 	if err != nil {
@@ -267,6 +273,7 @@ func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *ta
 	return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...)
 }
 
+// ExtractTo is just syntax sugar around ExtractReader
 func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
 	return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...)
 }