Implement compilation with images

A compilespec with an image defined (and no seed) can now be compiled,
and an artifact delta is provided at the end of the process.

pkg/compiler/compiler.go

@@ -18,6 +18,8 @@ package compiler
 import (
 	"errors"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -41,7 +43,7 @@ func NewLuetCompiler(backend CompilerBackend, t pkg.Tree) Compiler {
 	}
 }
 
-func (cs *LuetCompiler) Compile(p CompilationSpec) (Artifact, error) {
+func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
 	// on how we compute the resolvable tree.
@@ -55,42 +57,74 @@ func (cs *LuetCompiler) Compile(p CompilationSpec) (Artifact, error) {
 		p.SetSeedImage(p.GetImage()) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
 		// and we build all the images first.
 
+		buildDir := p.Rel("build")
+
+		// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
+		err := helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
+		if err != nil {
+			return nil, err
+
+		}
+
 		// First we create the builder image
-		p.WriteBuildImageDefinition(p.Rel(p.GetPackage().GetFingerPrint() + "-builder.dockerfile"))
+		p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
 		builderOpts := CompilerBackendOptions{
 			ImageName:      "luet/" + p.GetPackage().GetFingerPrint() + "-builder",
-			SourcePath:     p.GetOutputPath(),
-			DockerFileName: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.dockerfile"),
-			Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.rootfs.tar"),
+			SourcePath:     buildDir,
+			DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
+			Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 		}
-		err := cs.Backend.BuildImage(builderOpts)
+		err = cs.Backend.BuildImage(builderOpts)
+		if err != nil {
+			return nil, err
+		}
+
+		err = cs.Backend.ExportImage(builderOpts)
 		if err != nil {
 			return nil, err
 		}
 
 		// Then we write the step image, which uses the builder one
-		p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", p.Rel(p.GetPackage().GetFingerPrint()+".dockerfile"))
+		p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
 		runnerOpts := CompilerBackendOptions{
 			ImageName:      "luet/" + p.GetPackage().GetFingerPrint(),
-			SourcePath:     p.GetOutputPath(),
-			DockerFileName: p.Rel(p.GetPackage().GetFingerPrint() + ".dockerfile"),
-			Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".rootfs.tar"),
+			SourcePath:     buildDir,
+			DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
+			Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 		}
 		err = cs.Backend.ImageDefinitionToTar(runnerOpts)
 		if err != nil {
 			return nil, err
 		}
 
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, err
+		}
+
 		// TODO: Handle caching and optionally do not remove things
 		err = cs.Backend.RemoveImage(builderOpts)
 		if err != nil {
 			return nil, err
 		}
 
-		// TODO: Delta should be the artifact
-		return NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".rootfs.tar")), nil
+		rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		defer os.RemoveAll(rootfs) // clean up
+
+		// TODO: Compression and such
+		err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
+		if err != nil {
+			return nil, err
+		}
+		artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions)
+		if err != nil {
+			return nil, err
 		}
 
+		return artifact, nil
+	}
+
+	// TODO: Solve - hash
 	return nil, errors.New("Not implemented yet")
 }
 
@@ -100,6 +134,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	buildFile := pack.Rel(BuildFile)
 	if !helpers.Exists(buildFile) {
 		return nil, errors.New("No build file present for " + p.GetFingerPrint())
@@ -109,7 +144,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 	if err != nil {
 		return nil, err
 	}
-	return NewLuetCompilationSpec(dat, p)
+	return NewLuetCompilationSpec(dat, pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/interface.go

@@ -20,7 +20,7 @@ import (
 )
 
 type Compiler interface {
-	Compile(CompilationSpec) (Artifact, error)
+	Compile(int, bool, CompilationSpec) (Artifact, error)
 	FromPackage(pkg.Package) (CompilationSpec, error)
 
 	SetBackend(CompilerBackend)

pkg/helpers/archive.go

@@ -0,0 +1,62 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers
+
+import (
+	"io"
+	"os"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+func Tar(src, dest string) error {
+	out, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	fs, err := archive.Tar(src, archive.Uncompressed)
+	if err != nil {
+		return err
+	}
+	defer fs.Close()
+
+	_, err = io.Copy(out, fs)
+	if err != nil {
+		return err
+	}
+
+	err = out.Sync()
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+// Untar just a wrapper around the docker functions
+func Untar(src, dest string, sameOwner bool) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	return archive.Untar(in, dest, &archive.TarOptions{
+		NoLchown:        !sameOwner,
+		ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
+	})
+}

pkg/helpers/file.go

@@ -16,8 +16,11 @@
 package helpers
 
 import (
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 )
 
 // Exists reports whether the named file or directory exists.
@@ -37,3 +40,113 @@ func Read(file string) (string, error) {
 	}
 	return string(dat), nil
 }
+
+// CopyFile copies the contents of the file named src to the file named
+// by dst. The file will be created if it does not already exist. If the
+// destination file exists, all it's contents will be replaced by the contents
+// of the source file. The file mode will be copied from the source and
+// the copied data is synced/flushed to stable storage.
+func CopyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		if e := out.Close(); e != nil {
+			err = e
+		}
+	}()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return
+	}
+
+	err = out.Sync()
+	if err != nil {
+		return
+	}
+
+	si, err := os.Stat(src)
+	if err != nil {
+		return
+	}
+	err = os.Chmod(dst, si.Mode())
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func IsDirectory(path string) (bool, error) {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return false, err
+	}
+	return fileInfo.IsDir(), err
+}
+
+// CopyDir recursively copies a directory tree, attempting to preserve permissions.
+// Source directory must exist, destination directory must *not* exist.
+// Symlinks are ignored and skipped.
+func CopyDir(src string, dst string) (err error) {
+	src = filepath.Clean(src)
+	dst = filepath.Clean(dst)
+
+	si, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if !si.IsDir() {
+		return fmt.Errorf("source is not a directory")
+	}
+
+	_, err = os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return
+	}
+	if err == nil {
+		return fmt.Errorf("destination already exists")
+	}
+
+	err = os.MkdirAll(dst, si.Mode())
+	if err != nil {
+		return
+	}
+
+	entries, err := ioutil.ReadDir(src)
+	if err != nil {
+		return
+	}
+
+	for _, entry := range entries {
+		srcPath := filepath.Join(src, entry.Name())
+		dstPath := filepath.Join(dst, entry.Name())
+
+		if entry.IsDir() {
+			err = CopyDir(srcPath, dstPath)
+			if err != nil {
+				return
+			}
+		} else {
+			// Skip symlinks.
+			if entry.Mode()&os.ModeSymlink != 0 {
+				continue
+			}
+
+			err = CopyFile(srcPath, dstPath)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}

pkg/package/package.go

@@ -105,7 +105,7 @@ type DefaultPackage struct {
 	// TODO: Annotations?
 
 	// Path is set only internally when tree is loaded from disk
-	Path string `json:"path,omitempty"` // primary key with auto increment
+	Path string `json:"path,omitempty"`
 }
 
 // State represent the package state

pkg/tree/compiler_recipe.go

@@ -47,12 +47,13 @@ func (r *CompilerRecipe) Load(path string) error {
 		r.PackageTree = NewDefaultTree()
 	}
 
-	tmpfile, err := ioutil.TempFile("", "luet")
-	if err != nil {
-		return err
-	}
+	//tmpfile, err := ioutil.TempFile("", "luet")
+	//if err != nil {
+	//	return err
+	//}
 
-	r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
+	r.Tree().SetPackageSet(pkg.NewInMemoryDatabase(false))
+	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
 	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
 
 	// the function that handles each file or dir
@@ -97,7 +98,7 @@ func (r *CompilerRecipe) Load(path string) error {
 		return nil
 	}
 
-	err = filepath.Walk(path, ff)
+	err := filepath.Walk(path, ff)
 	if err != nil {
 		return err
 	}