⚙️ Add ability to build from Dockerfiles directly

pkg/api/core/types/artifact/artifact.go

@@ -43,8 +43,6 @@ import (
 	"github.com/mudler/luet/pkg/api/core/image"
 	"github.com/mudler/luet/pkg/api/core/types"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	compression "github.com/mudler/luet/pkg/compiler/types/compression"
-	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
@@ -58,17 +56,17 @@ import (
 type PackageArtifact struct {
 	Path string `json:"path"`
 
-	Dependencies      []*PackageArtifact                `json:"dependencies"`
-	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
-	Checksums         Checksums                         `json:"checksums"`
-	SourceAssertion   types.PackagesAssertions          `json:"-"`
-	CompressionType   compression.Implementation        `json:"compressiontype"`
-	Files             []string                          `json:"files"`
-	PackageCacheImage string                            `json:"package_cacheimage"`
-	Runtime           *types.Package                    `json:"runtime,omitempty"`
+	Dependencies      []*PackageArtifact              `json:"dependencies"`
+	CompileSpec       *types.LuetCompilationSpec      `json:"compilationspec"`
+	Checksums         Checksums                       `json:"checksums"`
+	SourceAssertion   types.PackagesAssertions        `json:"-"`
+	CompressionType   types.CompressionImplementation `json:"compressiontype"`
+	Files             []string                        `json:"files"`
+	PackageCacheImage string                          `json:"package_cacheimage"`
+	Runtime           *types.Package                  `json:"runtime,omitempty"`
 }
 
-func ImageToArtifact(ctx types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
+func ImageToArtifact(ctx types.Context, img v1.Image, t types.CompressionImplementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
 	_, tmpdiffs, err := image.Extract(ctx, img, filter)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -90,17 +88,12 @@ func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
 }
 
 func NewPackageArtifact(path string) *PackageArtifact {
-	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: types.None}
 }
 
 func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
 	p := &PackageArtifact{Checksums: Checksums{}}
-	err := yaml.Unmarshal(data, p)
-	if err != nil {
-		return p, err
-	}
-
-	return p, err
+	return p, yaml.Unmarshal(data, p)
 }
 
 func (a *PackageArtifact) Hash() error {
@@ -147,7 +140,6 @@ func (a *PackageArtifact) WriteYAML(dst string) error {
 	if err != nil {
 		return errors.Wrap(err, "Generated invalid artifact")
 	}
-
 	//p := a.CompileSpec.GetPackage().GetPath()
 
 	mangle.CompileSpec.GetPackage().SetPath("")
@@ -233,7 +225,7 @@ func (a *PackageArtifact) GenerateFinalImage(ctx types.Context, imageName string
 func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	switch a.CompressionType {
 
-	case compression.Zstandard:
+	case types.Zstandard:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -271,7 +263,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 		a.Path = zstdFile
 		return nil
-	case compression.GZip:
+	case types.GZip:
 		err := helpers.Tar(src, a.Path)
 		if err != nil {
 			return err
@@ -315,10 +307,10 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 
 func (a *PackageArtifact) getCompressedName() string {
 	switch a.CompressionType {
-	case compression.Zstandard:
+	case types.Zstandard:
 		return a.Path + ".zst"
 
-	case compression.GZip:
+	case types.GZip:
 		return a.Path + ".gz"
 	}
 	return a.Path
@@ -327,7 +319,7 @@ func (a *PackageArtifact) getCompressedName() string {
 // GetUncompressedName returns the artifact path without the extension suffix
 func (a *PackageArtifact) GetUncompressedName() string {
 	switch a.CompressionType {
-	case compression.Zstandard, compression.GZip:
+	case types.Zstandard, types.GZip:
 		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
 	}
 	return a.Path

pkg/api/core/types/artifact/artifact_test.go

@@ -21,14 +21,12 @@ import (
 	"path/filepath"
 
 	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/compiler"
 
 	"github.com/mudler/luet/pkg/api/core/context"
 	"github.com/mudler/luet/pkg/api/core/image"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
 	backend "github.com/mudler/luet/pkg/compiler/backend"
-	compression "github.com/mudler/luet/pkg/compiler/types/compression"
-	"github.com/mudler/luet/pkg/compiler/types/options"
-	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 
 	. "github.com/mudler/luet/pkg/compiler"
 	pkg "github.com/mudler/luet/pkg/database"
@@ -50,7 +48,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), options.WithContext(context.NewContext()))
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), compiler.WithContext(context.NewContext()))
 			lspec, err := cc.FromPackage(&types.Package{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -142,7 +140,7 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 
 			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
-			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Version: "1.0"}}
+			a.CompileSpec = &types.LuetCompilationSpec{Package: &types.Package{Name: "foo", Version: "1.0"}}
 
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
@@ -190,7 +188,7 @@ RUN echo bar > /test2`))
 			defer os.RemoveAll(tmpWork) // clean up
 
 			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
-			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Version: "1.0"}}
+			a.CompileSpec = &types.LuetCompilationSpec{Package: &types.Package{Name: "foo", Version: "1.0"}}
 
 			err = a.Compress(tmpdir, 1)
 			Expect(err).ToNot(HaveOccurred())
@@ -219,15 +217,15 @@ RUN echo bar > /test2`))
 
 		It("Retrieves uncompressed name", func() {
 			a := NewPackageArtifact("foo.tar.gz")
-			a.CompressionType = (compression.GZip)
+			a.CompressionType = (types.GZip)
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar.zst")
-			a.CompressionType = compression.Zstandard
+			a.CompressionType = types.Zstandard
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 
 			a = NewPackageArtifact("foo.tar")
-			a.CompressionType = compression.None
+			a.CompressionType = types.None
 			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
 		})
 	})

pkg/api/core/types/artifact/cache_test.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/mudler/luet/pkg/api/core/context"
 	. "github.com/mudler/luet/pkg/api/core/types/artifact"
-	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -77,13 +76,13 @@ var _ = Describe("Cache", func() {
 			_, err = cache.Get(a)
 			Expect(err).To(HaveOccurred())
 
-			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "bar"}}
+			a.CompileSpec = &types.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "bar"}}
 			_, _, err = cache.Put(a)
 			Expect(err).ToNot(HaveOccurred())
 
 			c := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
 			c.Hash()
-			c.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "bar"}}
+			c.CompileSpec = &types.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "bar"}}
 			_, err = cache.Get(c)
 			Expect(err).ToNot(HaveOccurred())
 		})

pkg/api/core/types/compilerspec.go

@@ -0,0 +1,454 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mitchellh/hashstructure/v2"
+
+	"github.com/ghodss/yaml"
+	"github.com/otiai10/copy"
+	dirhash "golang.org/x/mod/sumdb/dirhash"
+)
+
+type LuetCompilationspecs []LuetCompilationSpec
+
+func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
+	all := LuetCompilationspecs{}
+
+	for _, spec := range s {
+		all.Add(spec)
+	}
+	return &all
+}
+
+func (specs LuetCompilationspecs) Len() int {
+	return len(specs)
+}
+
+func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
+	newSpecs := LuetCompilationspecs{}
+SPECS:
+	for _, spec := range specs.All() {
+		for _, target := range s.All() {
+			if target.GetPackage().Matches(spec.GetPackage()) {
+				continue SPECS
+			}
+		}
+		newSpecs.Add(spec)
+	}
+	return &newSpecs
+}
+
+func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
+	*specs = append(*specs, *s)
+}
+
+func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
+	var cspecs []*LuetCompilationSpec
+	for i := range *specs {
+		f := (*specs)[i]
+		cspecs = append(cspecs, &f)
+	}
+
+	return cspecs
+}
+
+func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
+	newSpecs := LuetCompilationspecs{}
+	seen := map[string]bool{}
+
+	for i := range *specs {
+		j := (*specs)[i]
+		_, ok := seen[j.GetPackage().GetFingerPrint()]
+		if !ok {
+			seen[j.GetPackage().GetFingerPrint()] = true
+			newSpecs = append(newSpecs, j)
+		}
+	}
+	return &newSpecs
+}
+
+type CopyField struct {
+	Package     *Package `json:"package"`
+	Image       string   `json:"image"`
+	Source      string   `json:"source"`
+	Destination string   `json:"destination"`
+}
+
+type CompressionImplementation string
+
+const (
+	None      CompressionImplementation = "none" // e.g. tar for standard packages
+	GZip      CompressionImplementation = "gzip"
+	Zstandard CompressionImplementation = "zstd"
+)
+
+type LuetCompilationSpec struct {
+	Steps           []string           `json:"steps"` // Are run inside a container and the result layer diff is saved
+	Env             []string           `json:"env"`
+	Prelude         []string           `json:"prelude"` // Are run inside the image which will be our builder
+	Image           string             `json:"image"`
+	Seed            string             `json:"seed"`
+	Package         *Package           `json:"package"`
+	SourceAssertion PackagesAssertions `json:"-"`
+	PackageDir      string             `json:"package_dir" yaml:"package_dir"`
+
+	Retrieve []string `json:"retrieve"`
+
+	OutputPath string   `json:"-"` // Where the build processfiles go
+	Unpack     bool     `json:"unpack"`
+	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
+
+	BuildOptions *CompilerOptions `json:"build_options"`
+
+	Copy []CopyField `json:"copy"`
+
+	RequiresFinalImages bool `json:"requires_final_images" yaml:"requires_final_images"`
+}
+
+// Signature is a portion of the spec that yields a signature for the hash
+type Signature struct {
+	Image               string
+	Steps               []string
+	PackageDir          string
+	Prelude             []string
+	Seed                string
+	Env                 []string
+	Retrieve            []string
+	Unpack              bool
+	Includes            []string
+	Excludes            []string
+	Copy                []CopyField
+	Requires            Packages
+	RequiresFinalImages bool
+	Dockerfile          string
+}
+
+type CompilerOptions struct {
+	PushImageRepository      string
+	PullImageRepository      []string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          CompressionImplementation
+
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   LuetSolverOptions
+	BuildValuesFile []string
+	BuildValues     []map[string]interface{}
+
+	PackageTargetOnly bool
+	Rebuild           bool
+
+	BackendArgs []string
+
+	BackendType string
+
+	// TemplatesFolder. should default to tree/templates
+	TemplatesFolder []string
+
+	// Tells wether to push final container images after building
+	PushFinalImages      bool
+	PushFinalImagesForce bool
+
+	GenerateFinalImages bool
+
+	// Image repository to push to
+	PushFinalImagesRepository string
+	RuntimeDatabase           PackageDatabase
+
+	Context Context
+}
+
+type CompilerOption func(cfg *CompilerOptions) error
+
+func (cfg *CompilerOptions) Apply(opts ...CompilerOption) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cs *LuetCompilationSpec) signature() Signature {
+	return Signature{
+		Image:               cs.Image,
+		Steps:               cs.Steps,
+		PackageDir:          cs.PackageDir,
+		Prelude:             cs.Prelude,
+		Seed:                cs.Seed,
+		Env:                 cs.Env,
+		Retrieve:            cs.Retrieve,
+		Unpack:              cs.Unpack,
+		Includes:            cs.Includes,
+		Excludes:            cs.Excludes,
+		Copy:                cs.Copy,
+		Requires:            cs.Package.GetRequires(),
+		Dockerfile:          cs.Package.OriginDockerfile,
+		RequiresFinalImages: cs.RequiresFinalImages,
+	}
+}
+
+func NewLuetCompilationSpec(b []byte, p *Package) (*LuetCompilationSpec, error) {
+	var spec LuetCompilationSpec
+	var packageDefinition Package
+	err := yaml.Unmarshal(b, &spec)
+	if err != nil {
+		return &spec, err
+	}
+	err = yaml.Unmarshal(b, &packageDefinition)
+	if err != nil {
+		return &spec, err
+	}
+
+	// Update requires/conflict/provides
+	// When we have been passed a bytes slice, parse it as a package
+	// and updates requires/conflicts/provides.
+	// This is required in order to allow manipulation of such fields with templating
+	copy := *p
+	spec.Package = &copy
+	if len(packageDefinition.GetRequires()) != 0 {
+		spec.Package.Requires(packageDefinition.GetRequires())
+	}
+	if len(packageDefinition.GetConflicts()) != 0 {
+		spec.Package.Conflicts(packageDefinition.GetConflicts())
+	}
+	if len(packageDefinition.GetProvides()) != 0 {
+		spec.Package.SetProvides(packageDefinition.GetProvides())
+	}
+	return &spec, nil
+}
+func (cs *LuetCompilationSpec) GetSourceAssertion() PackagesAssertions {
+	return cs.SourceAssertion
+}
+
+func (cs *LuetCompilationSpec) SetBuildOptions(b CompilerOptions) {
+	cs.BuildOptions = &b
+}
+
+func (cs *LuetCompilationSpec) SetSourceAssertion(as PackagesAssertions) {
+	cs.SourceAssertion = as
+}
+func (cs *LuetCompilationSpec) GetPackage() *Package {
+	return cs.Package
+}
+
+func (cs *LuetCompilationSpec) GetPackageDir() string {
+	return cs.PackageDir
+}
+
+func (cs *LuetCompilationSpec) SetPackageDir(s string) {
+	cs.PackageDir = s
+}
+
+func (cs *LuetCompilationSpec) BuildSteps() []string {
+	return cs.Steps
+}
+
+func (cs *LuetCompilationSpec) ImageUnpack() bool {
+	return cs.Unpack
+}
+
+func (cs *LuetCompilationSpec) GetPreBuildSteps() []string {
+	return cs.Prelude
+}
+
+func (cs *LuetCompilationSpec) GetIncludes() []string {
+	return cs.Includes
+}
+
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
+func (cs *LuetCompilationSpec) GetRetrieve() []string {
+	return cs.Retrieve
+}
+
+// IsVirtual returns true if the spec is virtual.
+// A spec is virtual if the package is empty, and it has no image source to unpack from.
+func (cs *LuetCompilationSpec) IsVirtual() bool {
+	return cs.EmptyPackage() && !cs.HasImageSource()
+}
+
+func (cs *LuetCompilationSpec) GetSeedImage() string {
+	return cs.Seed
+}
+
+func (cs *LuetCompilationSpec) GetImage() string {
+	return cs.Image
+}
+
+func (cs *LuetCompilationSpec) GetOutputPath() string {
+	return cs.OutputPath
+}
+
+func (p *LuetCompilationSpec) Rel(s string) string {
+	return filepath.Join(p.GetOutputPath(), s)
+}
+
+func (cs *LuetCompilationSpec) SetImage(s string) {
+	cs.Image = s
+}
+
+func (cs *LuetCompilationSpec) SetOutputPath(s string) {
+	cs.OutputPath = s
+}
+
+func (cs *LuetCompilationSpec) SetSeedImage(s string) {
+	cs.Seed = s
+}
+
+func (cs *LuetCompilationSpec) EmptyPackage() bool {
+	return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage() && (cs.Package != nil && cs.Package.OriginDockerfile == "" || cs.Package == nil)
+}
+
+func (cs *LuetCompilationSpec) UnpackedPackage() bool {
+	// If package_dir was specified in the spec, we want to treat the content of the directory
+	// as the root of our archive.  ImageUnpack is implied to be true. override it
+	unpack := cs.ImageUnpack()
+	if cs.GetPackageDir() != "" {
+		unpack = true
+	}
+	return unpack
+}
+
+// HasImageSource returns true when the compilation spec has an image source.
+// a compilation spec has an image source when it depends on other packages or have a source image
+// explictly supplied
+func (cs *LuetCompilationSpec) HasImageSource() bool {
+	return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || (cs.RequiresFinalImages && len(cs.Package.GetRequires()) != 0)
+}
+
+func (cs *LuetCompilationSpec) Hash() (string, error) {
+	// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
+	signature := cs.signature()
+	h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
+	if err != nil {
+		return "", err
+	}
+	sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
+	if err != nil {
+		return fmt.Sprint(h), err
+	}
+	return fmt.Sprint(h, sum), err
+}
+
+func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
+	var err error
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			matches, err := filepath.Glob(cs.Rel(s))
+
+			if err != nil {
+				continue
+			}
+
+			for _, m := range matches {
+				err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
+			}
+		}
+	}
+	return err
+}
+
+func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
+	spec := `
+FROM ` + image + `
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=` + cs.Package.GetName() + `
+ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
+ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
+
+	if len(cs.Retrieve) > 0 {
+		for _, s := range cs.Retrieve {
+			//var file string
+			// if helpers.IsValidUrl(s) {
+			// 	file = s
+			// } else {
+			// 	file = cs.Rel(s)
+			// }
+			spec = spec + `
+ADD ` + s + ` /luetbuild/`
+		}
+	}
+
+	for _, c := range cs.Copy {
+		if c.Image != "" {
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
+		}
+	}
+
+	for _, s := range cs.Env {
+		spec = spec + `
+ENV ` + s
+	}
+
+	for _, s := range steps {
+		spec = spec + `
+RUN ` + s
+	}
+	return spec
+}
+
+// RenderBuildImage renders the dockerfile of the image used as a pre-build step
+func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+	return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
+
+}
+
+// RenderStepImage renders the dockerfile used for the image used for building the package
+func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
+	return cs.genDockerfile(image, cs.BuildSteps()), nil
+}
+
+func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {
+
+	data, err := cs.RenderBuildImage()
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, []byte(data), 0644)
+}
+
+func (cs *LuetCompilationSpec) WriteStepImageDefinition(fromimage, path string) error {
+	var data string
+	var err error
+	if cs.Package.OriginDockerfile != "" {
+		// pre-rendered
+		data = cs.Package.OriginDockerfile
+	} else {
+		data, err = cs.RenderStepImage(fromimage)
+		if err != nil {
+			return err
+		}
+	}
+
+	return ioutil.WriteFile(path, []byte(data), 0644)
+}

pkg/api/core/types/package.go

@@ -245,6 +245,8 @@ type Package struct {
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 
 	TreeDir string `json:"treedir,omitempty"`
+
+	OriginDockerfile string `json:"dockerfile,omitempty"`
 }
 
 // State represent the package state
@@ -267,6 +269,17 @@ func (p *Package) SetTreeDir(s string) {
 func (p *Package) GetTreeDir() string {
 	return p.TreeDir
 }
+
+func (p *Package) SetOriginalDockerfile(s string) error {
+	dat, err := ioutil.ReadFile(s)
+	if err != nil {
+		return errors.Wrap(err, "Error reading file "+s)
+	}
+
+	p.OriginDockerfile = string(dat)
+	return nil
+}
+
 func (p *Package) String() string {
 	b, err := p.JSON()
 	if err != nil {
@@ -712,6 +725,10 @@ func (p *Package) GetRuntimePackage() (*Package, error) {
 				break
 			}
 		}
+	} else if p.OriginDockerfile != "" {
+		// XXX: There are no runtime metadata at the moment available except package name in this case
+		// This needs to be adapted and aligned up with the tree parser
+		return &Package{Name: p.Name}, nil
 	} else {
 		definitionFile := filepath.Join(p.Path, PackageDefinitionFile)
 		dat, err := ioutil.ReadFile(definitionFile)

pkg/api/core/types/spec_test.go

@@ -0,0 +1,258 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/api/core/types"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	. "github.com/mudler/luet/pkg/compiler"
+	pkg "github.com/mudler/luet/pkg/database"
+	"github.com/mudler/luet/pkg/tree"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = ginkgo.Describe("Spec", func() {
+	ginkgo.Context("Luet specs", func() {
+		ginkgo.It("Allows normal operations", func() {
+			testSpec := &LuetCompilationSpec{Package: &Package{Name: "foo", Category: "a", Version: "0"}}
+			testSpec2 := &LuetCompilationSpec{Package: &Package{Name: "bar", Category: "a", Version: "0"}}
+			testSpec3 := &LuetCompilationSpec{Package: &Package{Name: "baz", Category: "a", Version: "0"}}
+			testSpec4 := &LuetCompilationSpec{Package: &Package{Name: "foo", Category: "a", Version: "0"}}
+
+			specs := NewLuetCompilationspecs(testSpec, testSpec2)
+			Expect(specs.Len()).To(Equal(2))
+			Expect(specs.All()).To(Equal([]*LuetCompilationSpec{testSpec, testSpec2}))
+			specs.Add(testSpec3)
+			Expect(specs.All()).To(Equal([]*LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
+			specs.Add(testSpec4)
+			Expect(specs.All()).To(Equal([]*LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
+			newSpec := specs.Unique()
+			Expect(newSpec.All()).To(Equal([]*LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
+
+			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
+			Expect(newSpec2.All()).To(Equal([]*LuetCompilationSpec{testSpec3}))
+
+		})
+		ginkgo.Context("virtuals", func() {
+			ginkgo.When("is empty", func() {
+				ginkgo.It("is virtual", func() {
+					spec := &LuetCompilationSpec{}
+					Expect(spec.IsVirtual()).To(BeTrue())
+				})
+			})
+			ginkgo.When("has defined steps", func() {
+				ginkgo.It("is not a virtual", func() {
+					spec := &LuetCompilationSpec{Steps: []string{"foo"}}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+			ginkgo.When("has defined image", func() {
+				ginkgo.It("is not a virtual", func() {
+					spec := &LuetCompilationSpec{Image: "foo"}
+					Expect(spec.IsVirtual()).To(BeFalse())
+				})
+			})
+		})
+	})
+
+	ginkgo.Context("Image hashing", func() {
+		ginkgo.It("is stable", func() {
+			spec1 := &LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &types.CompilerOptions{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+
+				Package: &Package{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec2 := &LuetCompilationSpec{
+				Image:        "foo",
+				BuildOptions: &types.CompilerOptions{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
+				Package: &Package{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			spec3 := &LuetCompilationSpec{
+				Image: "foo",
+				Steps: []string{"foo"},
+				Package: &Package{
+					Name:     "foo",
+					Category: "Bar",
+					Labels: map[string]string{
+						"foo": "bar",
+						"baz": "foo",
+					},
+				},
+			}
+			hash, err := spec1.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash2, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			hash3, err := spec3.Hash()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal(hash2))
+			hashagain, err := spec2.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(hash).ToNot(Equal(hash3))
+			Expect(hash).To(Equal(hashagain))
+		})
+	})
+
+	ginkgo.Context("Simple package build definition", func() {
+		ginkgo.It("Loads it correctly", func() {
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := compiler.FromPackage(&Package{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+			Expect(lspec.Image).To(Equal("luet/base"))
+			Expect(lspec.Seed).To(Equal("alpine"))
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			lspec.Env = []string{"test=1"}
+			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+ENV test=1`))
+
+			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+		})
+
+	})
+
+	ginkgo.It("Renders retrieve and env fields", func() {
+		generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+		err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+		compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+		lspec, err := compiler.FromPackage(&Package{Name: "a", Category: "test", Version: "1.0"})
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+		Expect(lspec.Image).To(Equal("luet/base"))
+		Expect(lspec.Seed).To(Equal("alpine"))
+		tmpdir, err := ioutil.TempDir("", "tree")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(tmpdir) // clean up
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		lspec.SetOutputPath("/foo/bar")
+
+		err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1`))
+
+		err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+		dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(dockerfile).To(Equal(`
+FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=a
+ENV PACKAGE_VERSION=1.0
+ENV PACKAGE_CATEGORY=test
+ADD test /luetbuild/
+ADD http://www.google.com /luetbuild/
+ENV test=1
+RUN echo foo > /test
+RUN echo bar > /test2`))
+
+	})
+})