Make DB Switchable

Fixes races conditions and make the DB Switchable. Also prepare inside
the CompilationSpec the tree of the deps to be built, and parallelize
only the building jobs.

Closes #7

Signed-off-by: Ettore Di Giacinto <mudler@gentoo.org>

Makefile

@@ -19,7 +19,7 @@ fmt:
 test:
 	GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo
 	GO111MODULE=off go get github.com/onsi/gomega/...
-	ginkgo -r ./...
+	ginkgo -race -r ./...
 
 .PHONY: coverage
 coverage:

cmd/build.go

@@ -61,6 +61,11 @@ var buildCmd = &cobra.Command{
 			Fatal("Error: " + err.Error())
 		}
 		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.Tree())
+
+		err = luetCompiler.Prepare(concurrency)
+		if err != nil {
+			Fatal("Error: " + err.Error())
+		}
 		if !all {
 			for _, a := range args {
 				decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)

pkg/compiler/artifact_test.go

@@ -34,7 +34,7 @@ var _ = Describe("Artifact", func() {
 	Context("Simple package build definition", func() {
 		It("Generates a delta", func() {
 
-			generalRecipe := tree.NewGeneralRecipe()
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
@@ -42,7 +42,7 @@ var _ = Describe("Artifact", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree())
+			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/backend/simpledocker_test.go

@@ -33,7 +33,7 @@ import (
 var _ = Describe("Docker backend", func() {
 	Context("Simple Docker backend satisfies main interface functionalities", func() {
 		It("Builds and generate tars", func() {
-			generalRecipe := tree.NewGeneralRecipe()
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
@@ -41,7 +41,7 @@ var _ = Describe("Docker backend", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree())
+			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/compiler/compiler.go

@@ -36,15 +36,17 @@ const BuildFile = "build.yaml"
 type LuetCompiler struct {
 	*tree.CompilerRecipe
 	Backend  CompilerBackend
+	Database pkg.PackageDatabase
 }
 
-func NewLuetCompiler(backend CompilerBackend, t pkg.Tree) Compiler {
+func NewLuetCompiler(backend CompilerBackend, t pkg.Tree, db pkg.PackageDatabase) Compiler {
 	// The CompilerRecipe will gives us a tree with only build deps listed.
 	return &LuetCompiler{
 		Backend: backend,
 		CompilerRecipe: &tree.CompilerRecipe{
 			tree.Recipe{PackageTree: t},
 		},
+		Database: db,
 	}
 }
 
@@ -52,7 +54,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.Compile(concurrency, keepPermissions, s)
+		ar, err := cs.compile(concurrency, keepPermissions, s)
 		if err != nil {
 			errors <- err
 		}
@@ -130,6 +132,11 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
 	}
 
 	for _, p := range ps.All() {
+		asserts, err := cs.ComputeDepTree(p)
+		if err != nil {
+			panic(err)
+		}
+		p.SetSourceAssertion(asserts)
 		all <- p
 	}
 
@@ -243,7 +250,14 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	artifact.SetCompileSpec(p)
 	return artifact, nil
 }
+func (cs *LuetCompiler) Prepare(concurrency int) error {
 
+	err := cs.Tree().ResolveDeps(concurrency) // FIXME: When done in parallel, this could be done on top before starting
+	if err != nil {
+		return errors.Wrap(err, "While resoolving tree world deps")
+	}
+	return nil
+}
 func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) {
 	pkgTag := "📦  " + p.GetPackage().GetName()
 
@@ -292,13 +306,59 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 	return artifact, nil
 }
 
-func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
-	Info("📦 Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... ☕")
+func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
 
-	err := cs.Tree().ResolveDeps(concurrency) // FIXME: When done in parallel, this could be done on top before starting
+	// Get build deps tree (ordered)
+	world, err := cs.Tree().World()
 	if err != nil {
-		return nil, errors.Wrap(err, "While resoolving tree world deps")
+		return nil, errors.Wrap(err, "While computing tree world")
 	}
+	s := solver.NewSolver([]pkg.Package{}, world, cs.Database)
+	pack, err := cs.Tree().FindPackage(p.GetPackage())
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+	}
+	solution, err := s.Install([]pkg.Package{pack})
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+	}
+	dependencies := solution.Order(p.GetPackage().GetFingerPrint())
+	assertions := solver.PackagesAssertions{}
+
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value && assertion.Package.Flagged() {
+			depPack, err := cs.Tree().FindPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+			}
+			nthsolution, err := s.Install([]pkg.Package{depPack})
+			if err != nil {
+				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
+			}
+
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.Order(depPack.GetFingerPrint()).Drop(depPack).AssertionHash(),
+				PackageHash: nthsolution.Order(depPack.GetFingerPrint()).AssertionHash(),
+			}
+			assertions = append(assertions, assertion)
+		}
+	}
+	p.SetSourceAssertion(assertions)
+	return assertions, nil
+}
+
+// Compile is non-parallel
+func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+	asserts, err := cs.ComputeDepTree(p)
+	if err != nil {
+		panic(err)
+	}
+	p.SetSourceAssertion(asserts)
+	return cs.compile(concurrency, keepPermissions, p)
+}
+
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
+	Info("📦 Compiling", p.GetPackage().GetName(), "version", p.GetPackage().GetVersion(), ".... ☕")
 
 	if len(p.GetPackage().GetRequires()) == 0 && p.GetImage() == "" {
 		Error("Package with no deps and no seed image supplied, bailing out")
@@ -306,7 +366,6 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 	}
 
 	// - If image is set we just generate a plain dockerfile
-
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
 		if p.ImageUnpack() { // If it is just an entire image, create a package from it
@@ -321,24 +380,7 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 	// This means to recursively build all the build-images needed to reach that tree part.
 	// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
 
-	// Get build deps tree (ordered)
-	world, err := cs.Tree().World()
-	if err != nil {
-		return nil, errors.Wrap(err, "While computing tree world")
-	}
-
-	s := solver.NewSolver([]pkg.Package{}, world)
-	pack, err := cs.Tree().FindPackage(p.GetPackage())
-	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-	}
-	solution, err := s.Install([]pkg.Package{pack})
-	if err != nil {
-		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-	}
-
-	allOrderedDeps := solution.Order(p.GetPackage().GetFingerPrint())
-	dependencies := allOrderedDeps.Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
 	departifacts := []Artifact{}                                // TODO: Return this somehow
 	deperrs := []error{}
 	var lastHash string
@@ -347,14 +389,12 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 
 	Info("🌲 Build dependencies for " + p.GetPackage().GetName())
 	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value && assertion.Package.Flagged() {
 		depsN++
 		Info(" ⤷", assertion.Package.GetName(), "🍃", assertion.Package.GetVersion(), "(", assertion.Package.GetCategory(), ")")
-		}
+
 	}
 
 	for _, assertion := range dependencies { //highly dependent on the order
-		if assertion.Value && assertion.Package.Flagged() {
 		currentN++
 		pkgTag := fmt.Sprintf("📦  %d/%d %s ⤑ %s", currentN, depsN, p.GetPackage().GetName(), assertion.Package.GetName())
 		Info(pkgTag, "   🏗  Building dependency")
@@ -363,19 +403,9 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 			return nil, errors.New("Error while generating compilespec for " + assertion.Package.GetName())
 		}
 		compileSpec.SetOutputPath(p.GetOutputPath())
-			depPack, err := cs.Tree().FindPackage(assertion.Package)
-			if err != nil {
-				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-			}
 
-			// Generate image name of builder image - it should match with the hash up to this point - package
-			nthsolution, err := s.Install([]pkg.Package{depPack})
-			if err != nil {
-				return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
-			}
-
-			buildImageHash := "luet/cache:" + nthsolution.Order(depPack.GetFingerPrint()).Drop(depPack).AssertionHash()
-			currentPackageImageHash := "luet/cache:" + nthsolution.Order(depPack.GetFingerPrint()).AssertionHash()
+		buildImageHash := "luet/cache:" + assertion.Hash.BuildHash
+		currentPackageImageHash := "luet/cache:" + assertion.Hash.PackageHash
 		Debug(pkgTag, "    ⤷ 🐋 Builder image name", buildImageHash)
 		Debug(pkgTag, "    ⤷ 🐋 Package image name", currentPackageImageHash)
 
@@ -412,14 +442,13 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 		departifacts = append(departifacts, artifact)
 		Info(pkgTag, "💥 Done")
 	}
-	}
 	Info("📦", p.GetPackage().GetName(), "🌪  Building package target from:", lastHash)
 	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
 	if err != nil {
 		return artifact, err
 	}
 	artifact.SetDependencies(departifacts)
-	artifact.SetSourceAssertion(allOrderedDeps)
+	artifact.SetSourceAssertion(p.GetSourceAssertion())
 
 	return artifact, err
 }

pkg/compiler/compiler_test.go

@@ -31,7 +31,7 @@ import (
 var _ = Describe("Compiler", func() {
 	Context("Simple package build definition", func() {
 		It("Compiles it correctly", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
@@ -39,7 +39,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -73,7 +76,7 @@ var _ = Describe("Compiler", func() {
 
 	Context("Simple package build definition", func() {
 		It("Compiles it in parallel", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../tests/fixtures/buildable")
 			Expect(err).ToNot(HaveOccurred())
@@ -81,7 +84,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
@@ -107,7 +113,7 @@ var _ = Describe("Compiler", func() {
 
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -118,7 +124,11 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
@@ -163,7 +173,7 @@ var _ = Describe("Compiler", func() {
 		})
 
 		It("unpacks images when needed", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -174,7 +184,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(2))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"})
@@ -205,7 +218,7 @@ var _ = Describe("Compiler", func() {
 		})
 
 		It("Compiles and includes ony wanted files", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -216,7 +229,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -239,7 +255,7 @@ var _ = Describe("Compiler", func() {
 		})
 
 		It("Compiles a more complex tree", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -250,7 +266,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -277,7 +296,7 @@ var _ = Describe("Compiler", func() {
 		})
 
 		It("Compiles revdeps", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "revdep")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -288,7 +307,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
 			Expect(err).ToNot(HaveOccurred())
 
@@ -315,7 +337,7 @@ var _ = Describe("Compiler", func() {
 		})
 
 		It("Compiles revdeps with seeds", func() {
-			generalRecipe := tree.NewCompilerRecipe()
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
@@ -326,7 +348,10 @@ var _ = Describe("Compiler", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4))
 
-			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
 
 			spec.SetOutputPath(tmpdir)

pkg/compiler/interface.go

@@ -24,6 +24,8 @@ type Compiler interface {
 	Compile(int, bool, CompilationSpec) (Artifact, error)
 	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
 	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
+	Prepare(concurrency int) error
 
 	FromPackage(pkg.Package) (CompilationSpec, error)
 
@@ -102,6 +104,9 @@ type CompilationSpec interface {
 	Rel(string) string
 
 	GetPreBuildSteps() []string
+
+	GetSourceAssertion() solver.PackagesAssertions
+	SetSourceAssertion(as solver.PackagesAssertions)
 }
 
 type CompilationSpecs interface {

pkg/compiler/spec.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -91,6 +92,8 @@ type LuetCompilationSpec struct {
 	Image           string                    `json:"image"`
 	Seed            string                    `json:"seed"`
 	Package         pkg.Package               `json:"-"`
+	SourceAssertion solver.PackagesAssertions `json:"-"`
+
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
@@ -105,7 +108,13 @@ func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
 	spec.Package = p
 	return &spec, nil
 }
+func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
+	return a.SourceAssertion
+}
 
+func (a *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
+	a.SourceAssertion = as
+}
 func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
 	return cs.Package
 }

pkg/compiler/spec_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Spec", func() {
 
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
-			generalRecipe := tree.NewGeneralRecipe()
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
@@ -62,7 +62,7 @@ var _ = Describe("Spec", func() {
 
 			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree())
+			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
 			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 

pkg/package/database_mem.go

@@ -24,7 +24,10 @@ import (
 	"sync"
 )
 
-var DBInMemoryInstance PackageDatabase
+var DBInMemoryInstance = &InMemoryDatabase{
+	Mutex: &sync.Mutex{},
+
+	Database: map[string]string{}}
 
 type InMemoryDatabase struct {
 	*sync.Mutex
@@ -33,12 +36,6 @@ type InMemoryDatabase struct {
 
 func NewInMemoryDatabase(singleton bool) PackageDatabase {
 	// In memoryDB is a singleton
-	if singleton && DBInMemoryInstance == nil {
-		DBInMemoryInstance = &InMemoryDatabase{
-			Mutex: &sync.Mutex{},
-
-			Database: map[string]string{}}
-	}
 	if !singleton {
 		return &InMemoryDatabase{
 			Mutex: &sync.Mutex{},

pkg/package/package.go

@@ -29,9 +29,9 @@ import (
 // Package is a package interface (TBD)
 // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package
 type Package interface {
-	Encode() (string, error)
+	Encode(PackageDatabase) (string, error)
 
-	BuildFormula() ([]bf.Formula, error)
+	BuildFormula(PackageDatabase) ([]bf.Formula, error)
 	IsFlagged(bool) Package
 	Flagged() bool
 	GetFingerPrint() string
@@ -74,9 +74,9 @@ type PackageSet interface {
 }
 
 type Tree interface {
-	GetPackageSet() PackageSet
+	GetPackageSet() PackageDatabase
 	Prelude() string // A tree might have a prelude to be able to consume a tree
-	SetPackageSet(s PackageSet)
+	SetPackageSet(s PackageDatabase)
 	World() ([]Package, error)
 	FindPackage(Package) (Package, error)
 	ResolveDeps(int) error
@@ -161,8 +161,8 @@ func (p *DefaultPackage) RemoveUse(use string) {
 
 // Encode encodes the package to string.
 // It returns an ID which can be used to retrieve the package later on.
-func (p *DefaultPackage) Encode() (string, error) {
-	return NewInMemoryDatabase(true).CreatePackage(p)
+func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) {
+	return db.CreatePackage(p)
 }
 
 func (p *DefaultPackage) Yaml() ([]byte, error) {
@@ -270,8 +270,8 @@ func (p *DefaultPackage) Revdeps(world *[]Package) []Package {
 	return versionsInWorld
 }
 
-func DecodePackage(ID string) (Package, error) {
-	return NewInMemoryDatabase(true).GetPackage(ID)
+func DecodePackage(ID string, db PackageDatabase) (Package, error) {
+	return db.GetPackage(ID)
 }
 
 func NormalizeFlagged(p Package) {
@@ -299,8 +299,8 @@ func (p *DefaultPackage) RequiresContains(s Package) bool {
 	return false
 }
 
-func (p *DefaultPackage) BuildFormula() ([]bf.Formula, error) {
-	encodedA, err := p.IsFlagged(true).Encode()
+func (p *DefaultPackage) BuildFormula(db PackageDatabase) ([]bf.Formula, error) {
+	encodedA, err := p.IsFlagged(true).Encode(db)
 	if err != nil {
 		return nil, err
 	}
@@ -311,14 +311,14 @@ func (p *DefaultPackage) BuildFormula() ([]bf.Formula, error) {
 	var formulas []bf.Formula
 
 	for _, required := range p.PackageRequires {
-		encodedB, err := required.Encode()
+		encodedB, err := required.Encode(db)
 		if err != nil {
 			return nil, err
 		}
 		B := bf.Var(encodedB)
 		formulas = append(formulas, bf.Or(bf.Not(A), B))
 
-		f, err := required.BuildFormula()
+		f, err := required.BuildFormula(db)
 		if err != nil {
 			return nil, err
 		}
@@ -327,7 +327,7 @@ func (p *DefaultPackage) BuildFormula() ([]bf.Formula, error) {
 	}
 
 	for _, required := range p.PackageConflicts {
-		encodedB, err := required.Encode()
+		encodedB, err := required.Encode(db)
 		if err != nil {
 			return nil, err
 		}
@@ -335,7 +335,7 @@ func (p *DefaultPackage) BuildFormula() ([]bf.Formula, error) {
 		formulas = append(formulas, bf.Or(bf.Not(A),
 			bf.Not(B)))
 
-		f, err := required.BuildFormula()
+		f, err := required.BuildFormula(db)
 		if err != nil {
 			return nil, err
 		}

pkg/solver/decoder.go

@@ -28,19 +28,25 @@ import (
 
 type PackagesAssertions []PackageAssert
 
+type PackageHash struct {
+	BuildHash   string
+	PackageHash string
+}
+
 // PackageAssert represent a package assertion.
 // It is composed of a Package and a Value which is indicating the absence or not
 // of the associated package state.
 type PackageAssert struct {
 	Package pkg.Package
 	Value   bool
+	Hash    PackageHash
 }
 
 // DecodeModel decodes a model from the SAT solver to package assertions (PackageAssert)
-func DecodeModel(model map[string]bool) (PackagesAssertions, error) {
+func DecodeModel(model map[string]bool, db pkg.PackageDatabase) (PackagesAssertions, error) {
 	ass := make(PackagesAssertions, 0)
 	for k, v := range model {
-		a, err := pkg.DecodePackage(k)
+		a, err := pkg.DecodePackage(k, db)
 		if err != nil {
 			return nil, err
 

pkg/solver/solver.go

@@ -33,6 +33,7 @@ type PackageSolver interface {
 
 // Solver is the default solver for luet
 type Solver struct {
+	Database  pkg.PackageDatabase
 	Wanted    []pkg.Package
 	Installed []pkg.Package
 	World     []pkg.Package
@@ -40,14 +41,14 @@ type Solver struct {
 
 // NewSolver accepts as argument two lists of packages, the first is the initial set,
 // the second represent all the known packages.
-func NewSolver(init []pkg.Package, w []pkg.Package) PackageSolver {
+func NewSolver(init []pkg.Package, w []pkg.Package , db pkg.PackageDatabase) PackageSolver {
 	for _, v := range init {
 		pkg.NormalizeFlagged(v)
 	}
 	for _, v := range w {
 		pkg.NormalizeFlagged(v)
 	}
-	return &Solver{Installed: init, World: w}
+	return &Solver{Installed: init, World: w, Database: db}
 }
 
 // SetWorld is a setter for the list of all known packages to the solver
@@ -69,7 +70,7 @@ func (s *Solver) noRulesWorld() bool {
 func (s *Solver) BuildInstalled() (bf.Formula, error) {
 	var formulas []bf.Formula
 	for _, p := range s.Installed {
-		solvable, err := p.BuildFormula()
+		solvable, err := p.BuildFormula(s.Database)
 		if err != nil {
 			return nil, err
 		}
@@ -96,7 +97,7 @@ func (s *Solver) BuildWorld(includeInstalled bool) (bf.Formula, error) {
 	}
 
 	for _, p := range s.World {
-		solvable, err := p.BuildFormula()
+		solvable, err := p.BuildFormula(s.Database)
 		if err != nil {
 			return nil, err
 		}
@@ -113,7 +114,7 @@ func (s *Solver) ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error) {
 		return false, nil
 	}
 
-	encodedP, err := p.IsFlagged(true).Encode()
+	encodedP, err := p.IsFlagged(true).Encode(s.Database)
 	if err != nil {
 		return false, err
 	}
@@ -135,7 +136,7 @@ func (s *Solver) ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error) {
 		// 	continue
 		// }
 
-		encodedI, err := i.Encode()
+		encodedI, err := i.Encode(s.Database)
 		if err != nil {
 			return false, err
 		}
@@ -215,7 +216,7 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 		return nil, err
 	}
 	for _, wanted := range s.Wanted {
-		encodedW, err := wanted.Encode()
+		encodedW, err := wanted.Encode(s.Database)
 		if err != nil {
 			return nil, err
 		}
@@ -227,7 +228,7 @@ func (s *Solver) BuildFormula() (bf.Formula, error) {
 		}
 
 		for _, installed := range s.Installed {
-			encodedI, err := installed.Encode()
+			encodedI, err := installed.Encode(s.Database)
 			if err != nil {
 				return nil, err
 			}
@@ -252,7 +253,6 @@ func (s *Solver) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
 
 // Solve builds the formula given the current state and returns package assertions
 func (s *Solver) Solve() (PackagesAssertions, error) {
-
 	f, err := s.BuildFormula()
 
 	if err != nil {
@@ -264,7 +264,7 @@ func (s *Solver) Solve() (PackagesAssertions, error) {
 		return nil, err
 	}
 
-	return DecodeModel(model)
+	return DecodeModel(model, s.Database)
 }
 
 // Install given a list of packages, returns package assertions to indicate the packages that must be installed in the system in order

pkg/tree/compiler_recipe.go

@@ -34,7 +34,9 @@ const (
 	CompilerDefinitionFile = "build.yaml"
 )
 
-func NewCompilerRecipe() Builder { return &CompilerRecipe{} }
+func NewCompilerRecipe(d pkg.PackageDatabase) Builder {
+	return &CompilerRecipe{Recipe: Recipe{Database: d}}
+}
 
 // Recipe is the "general" reciper for Trees
 type CompilerRecipe struct {
@@ -52,7 +54,7 @@ func (r *CompilerRecipe) Load(path string) error {
 	//	return err
 	//}
 
-	r.Tree().SetPackageSet(pkg.NewInMemoryDatabase(false))
+	r.Tree().SetPackageSet(r.Database)
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
 	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()
 	// the function that handles each file or dir

pkg/tree/recipes.go

@@ -33,11 +33,12 @@ const (
 	DefinitionFile = "definition.yaml"
 )
 
-func NewGeneralRecipe() Builder { return &Recipe{} }
+func NewGeneralRecipe(db pkg.PackageDatabase) Builder { return &Recipe{Database: db} }
 
 // Recipe is the "general" reciper for Trees
 type Recipe struct {
 	PackageTree pkg.Tree
+	Database    pkg.PackageDatabase
 }
 
 func (r *Recipe) Save(path string) error {
@@ -73,7 +74,7 @@ func (r *Recipe) Load(path string) error {
 	// if err != nil {
 	// 	return err
 	// }
-	r.Tree().SetPackageSet(pkg.NewInMemoryDatabase(false))
+	r.Tree().SetPackageSet(r.Database)
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))
 	// TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean()

pkg/tree/recipes_test.go

@@ -117,7 +117,7 @@ var _ = Describe("Recipe", func() {
 				world, err := tree.World()
 				Expect(err).ToNot(HaveOccurred())
 
-				s := solver.NewSolver([]pkg.Package{}, world)
+				s := solver.NewSolver([]pkg.Package{}, world, tree.GetPackageSet())
 				solution, err := s.Install([]pkg.Package{pack})
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(solution)).To(Equal(3))

pkg/tree/tree.go

@@ -32,11 +32,12 @@ import (
 func NewDefaultTree() pkg.Tree { return &DefaultTree{} }
 
 type DefaultTree struct {
-	Packages   pkg.PackageSet
+	sync.Mutex
+	Packages   pkg.PackageDatabase
 	CacheWorld []pkg.Package
 }
 
-func (gt *DefaultTree) GetPackageSet() pkg.PackageSet {
+func (gt *DefaultTree) GetPackageSet() pkg.PackageDatabase {
 	return gt.Packages
 }
 
@@ -44,11 +45,13 @@ func (gt *DefaultTree) Prelude() string {
 	return ""
 }
 
-func (gt *DefaultTree) SetPackageSet(s pkg.PackageSet) {
+func (gt *DefaultTree) SetPackageSet(s pkg.PackageDatabase) {
 	gt.Packages = s
 }
 
 func (gt *DefaultTree) World() ([]pkg.Package, error) {
+	gt.Lock()
+	defer gt.Unlock()
 	if len(gt.CacheWorld) > 0 {
 		return gt.CacheWorld, nil
 	}
@@ -66,6 +69,8 @@ func (gt *DefaultTree) World() ([]pkg.Package, error) {
 }
 
 func (gt *DefaultTree) UpdateWorldPackage(p pkg.Package) {
+	gt.Lock()
+	defer gt.Unlock()
 	//var CacheWorld []pkg.Package
 	for _, pid := range gt.CacheWorld {
 		if p.Matches(pid) {