Add revdep calculation

Add CompilationSpecs type to handle slices of CompilationSpec, to
perform operation such as Unique() and Remove().

Add also dependencies and the spec associated to the Artifact, to track
how the artifact was generated.

Add revdeps compilation wrapper, and unit tests

pkg/compiler/artifact.go

@@ -25,17 +25,46 @@ import (
 	"sync"
 
 	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/pkg/solver"
 
 	"github.com/mudler/luet/pkg/helpers"
 	"github.com/pkg/errors"
 )
 
 type PackageArtifact struct {
-	Path string
+	Path         string
+	Dependencies []Artifact
+	CompileSpec  CompilationSpec
+
+	SourceAssertion solver.PackagesAssertions
 }
 
 func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path}
+	return &PackageArtifact{Path: path, Dependencies: []Artifact{}}
+}
+
+func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
+	return a.SourceAssertion
+}
+
+func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
+	a.CompileSpec = as
+}
+
+func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
+	return a.CompileSpec
+}
+
+func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
+	a.SourceAssertion = as
+}
+
+func (a *PackageArtifact) GetDependencies() []Artifact {
+	return a.Dependencies
+}
+
+func (a *PackageArtifact) SetDependencies(d []Artifact) {
+	a.Dependencies = d
 }
 
 func (a *PackageArtifact) GetPath() string {

pkg/compiler/compiler.go

@@ -62,19 +62,74 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
 		m.Unlock()
 	}
 }
+func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
+	artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps)
+	if len(err) != 0 {
+		return artifacts, err
+	}
 
-func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps []CompilationSpec) ([]Artifact, []error) {
+	Info("🌲 Resolving reverse dependencies")
+	toCompile := NewLuetCompilationspecs()
+	for _, a := range artifacts {
+		w, asserterr := cs.Tree().World()
+		if asserterr != nil {
+			return nil, append(err, asserterr)
+		}
+		revdeps := a.GetCompileSpec().GetPackage().Revdeps(&w)
+		for _, r := range revdeps {
+			spec, asserterr := cs.FromPackage(r)
+			if err != nil {
+				return nil, append(err, asserterr)
+			}
+			spec.SetOutputPath(ps.All()[0].GetOutputPath())
+
+			toCompile.Add(spec)
+		}
+		// for _, assertion := range a.GetSourceAssertion() {
+		// 	if assertion.Value && assertion.Package.Flagged() {
+		// 		spec, asserterr := cs.FromPackage(assertion.Package)
+		// 		if err != nil {
+		// 			return nil, append(err, asserterr)
+		// 		}
+		// 		w, asserterr := cs.Tree().World()
+		// 		if err != nil {
+		// 			return nil, append(err, asserterr)
+		// 		}
+		// 		revdeps := spec.GetPackage().Revdeps(&w)
+		// 		for _, r := range revdeps {
+		// 			spec, asserterr := cs.FromPackage(r)
+		// 			if asserterr != nil {
+		// 				return nil, append(err, asserterr)
+		// 			}
+		// 			spec.SetOutputPath(ps.All()[0].GetOutputPath())
+
+		// 			toCompile.Add(spec)
+		// 		}
+		// 	}
+		// }
+	}
+
+	uniques := toCompile.Unique().Remove(ps)
+	for _, u := range uniques.All() {
+		Info(" ⤷", u.GetPackage().GetName(), "🍃", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
+	}
+
+	artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques)
+	return append(artifacts, artifacts2...), err
+}
+
+func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
 	all := make(chan CompilationSpec)
 	artifacts := []Artifact{}
 	mutex := &sync.Mutex{}
-	errors := make(chan error, len(ps))
+	errors := make(chan error, ps.Len())
 	var wg = new(sync.WaitGroup)
 	for i := 0; i < concurrency; i++ {
 		wg.Add(1)
 		go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors)
 	}
 
-	for _, p := range ps {
+	for _, p := range ps.All() {
 		all <- p
 	}
 
@@ -185,7 +240,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not generate deltas")
 	}
-
+	artifact.SetCompileSpec(p)
 	return artifact, nil
 }
 
@@ -232,7 +287,9 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
 	}
 
 	Info(pkgTag, "   🎉 Done")
-	return NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")), nil
+	artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
+	artifact.SetCompileSpec(p)
+	return artifact, nil
 }
 
 func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
@@ -280,8 +337,9 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().GetName())
 	}
 
-	dependencies := solution.Order(p.GetPackage().GetFingerPrint()).Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
-	departifacts := []Artifact{}                                                         // TODO: Return this somehow
+	allOrderedDeps := solution.Order(p.GetPackage().GetFingerPrint())
+	dependencies := allOrderedDeps.Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
+	departifacts := []Artifact{}                        // TODO: Return this somehow
 	deperrs := []error{}
 	var lastHash string
 	depsN := 0
@@ -356,8 +414,14 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 		}
 	}
 	Info("📦", p.GetPackage().GetName(), "🌪  Building package target from:", lastHash)
+	artifact, err := cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
+	if err != nil {
+		return artifact, err
+	}
+	artifact.SetDependencies(departifacts)
+	artifact.SetSourceAssertion(allOrderedDeps)
 
-	return cs.compileWithImage(lastHash, "", "", concurrency, keepPermissions, p)
+	return artifact, err
 }
 
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {

pkg/compiler/compiler_test.go

@@ -95,7 +95,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
-			artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2})
+			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2))
 			Expect(len(errs)).To(Equal(0))
 			for _, artifact := range artifacts {
 				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@@ -135,7 +135,7 @@ var _ = Describe("Compiler", func() {
 			spec2.SetOutputPath(tmpdir)
 			spec3.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(2, false, []CompilationSpec{spec, spec2, spec3})
+			artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2, spec3))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(3))
 
@@ -182,11 +182,11 @@ var _ = Describe("Compiler", func() {
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec})
+			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
-			artifacts2, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec2})
+			artifacts2, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec2))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts2)).To(Equal(1))
 
@@ -225,7 +225,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec})
+			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -259,7 +259,7 @@ var _ = Describe("Compiler", func() {
 
 			spec.SetOutputPath(tmpdir)
 
-			artifacts, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec})
+			artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec))
 			Expect(errs).To(BeNil())
 			Expect(len(artifacts)).To(Equal(1))
 
@@ -276,5 +276,95 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Compiles revdeps", func() {
+			generalRecipe := tree.NewCompilerRecipe()
+			tmpdir, err := ioutil.TempDir("", "revdep")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/layered")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
+
+			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(2))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
+
+			Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
+		})
+
+		It("Compiles revdeps with seeds", func() {
+			generalRecipe := tree.NewCompilerRecipe()
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/buildableseed")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
+
+			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(4))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+
+			spec.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(4))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+
+			// A deps on B, so A artifacts are here:
+			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+
+			// B
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("artifact42"))).To(BeTrue())
+
+			// C depends on B, so B is here
+			content1, err := helpers.Read(spec.Rel("c"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("cd"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("c\n"))
+			Expect(content2).To(Equal("c\n"))
+
+			// D is here as it requires C, and C was recompiled
+			content1, err = helpers.Read(spec.Rel("d"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err = helpers.Read(spec.Rel("dd"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("s\n"))
+			Expect(content2).To(Equal("dd\n"))
+		})
+
 	})
 })

pkg/compiler/interface.go

@@ -17,11 +17,14 @@ package compiler
 
 import (
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 )
 
 type Compiler interface {
 	Compile(int, bool, CompilationSpec) (Artifact, error)
-	CompileParallel(concurrency int, keepPermissions bool, ps []CompilationSpec) ([]Artifact, []error)
+	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
+
 	FromPackage(pkg.Package) (CompilationSpec, error)
 
 	SetBackend(CompilerBackend)
@@ -50,6 +53,13 @@ type CompilerBackend interface {
 type Artifact interface {
 	GetPath() string
 	SetPath(string)
+	GetDependencies() []Artifact
+	SetDependencies(d []Artifact)
+	GetSourceAssertion() solver.PackagesAssertions
+	SetSourceAssertion(as solver.PackagesAssertions)
+
+	SetCompileSpec(as CompilationSpec)
+	GetCompileSpec() CompilationSpec
 }
 
 type ArtifactNode struct {
@@ -93,3 +103,11 @@ type CompilationSpec interface {
 
 	GetPreBuildSteps() []string
 }
+
+type CompilationSpecs interface {
+	Unique() CompilationSpecs
+	Len() int
+	All() []CompilationSpec
+	Add(CompilationSpec)
+	Remove(s CompilationSpecs) CompilationSpecs
+}

pkg/compiler/spec.go

@@ -23,6 +23,68 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
+type LuetCompilationspecs []LuetCompilationSpec
+
+func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
+	all := LuetCompilationspecs{}
+
+	for _, spec := range s {
+		all.Add(spec)
+	}
+	return &all
+}
+
+func (specs LuetCompilationspecs) Len() int {
+	return len(specs)
+}
+
+func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
+	newSpecs := LuetCompilationspecs{}
+SPECS:
+	for _, spec := range specs.All() {
+		for _, target := range s.All() {
+			if target.GetPackage().Matches(spec.GetPackage()) {
+				continue SPECS
+			}
+		}
+		newSpecs.Add(spec)
+	}
+	return &newSpecs
+}
+
+func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
+	c, ok := s.(*LuetCompilationSpec)
+	if !ok {
+		panic("LuetCompilationspecs supports only []LuetCompilationSpec")
+	}
+	*specs = append(*specs, *c)
+}
+
+func (specs *LuetCompilationspecs) All() []CompilationSpec {
+	var cspecs []CompilationSpec
+	for i, _ := range *specs {
+		f := (*specs)[i]
+		cspecs = append(cspecs, &f)
+	}
+
+	return cspecs
+}
+
+func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
+	newSpecs := LuetCompilationspecs{}
+	seen := map[string]bool{}
+
+	for i, _ := range *specs {
+		j := (*specs)[i]
+		_, ok := seen[j.GetPackage().GetFingerPrint()]
+		if !ok {
+			seen[j.GetPackage().GetFingerPrint()] = true
+			newSpecs = append(newSpecs, j)
+		}
+	}
+	return &newSpecs
+}
+
 type LuetCompilationSpec struct {
 	Steps      []string    `json:"steps"`   // Are run inside a container and the result layer diff is saved
 	Prelude    []string    `json:"prelude"` // Are run inside the image which will be our builder

pkg/compiler/spec_test.go

@@ -29,6 +29,29 @@ import (
 )
 
 var _ = Describe("Spec", func() {
+	Context("Luet specs", func() {
+		It("Allows normal operations", func() {
+			testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+			testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
+			testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
+			testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
+
+			specs := NewLuetCompilationspecs(testSpec, testSpec2)
+			Expect(specs.Len()).To(Equal(2))
+			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
+			specs.Add(testSpec3)
+			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+			specs.Add(testSpec4)
+			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
+			newSpec := specs.Unique()
+			Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
+
+			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
+			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
+
+		})
+	})
+
 	Context("Simple package build definition", func() {
 		It("Loads it correctly", func() {
 			generalRecipe := tree.NewGeneralRecipe()

pkg/package/package_test.go

@@ -56,7 +56,7 @@ var _ = Describe("Package", func() {
 		b := NewPackage("B", "1.0", []*DefaultPackage{a}, []*DefaultPackage{})
 		c := NewPackage("C", "1.1", []*DefaultPackage{b}, []*DefaultPackage{})
 		d := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
-		e := NewPackage("D", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
+		e := NewPackage("E", "0.1", []*DefaultPackage{c}, []*DefaultPackage{})
 
 		It("Computes correctly", func() {
 			lst := b.Revdeps(&[]Package{a, b, c, d, e})

pkg/solver/decoder.go

@@ -18,11 +18,12 @@ package solver
 import (
 	"crypto/sha256"
 	"fmt"
+	"sort"
+	"unicode"
+
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/philopon/go-toposort"
 	"github.com/stevenle/topsort"
-	"sort"
-	"unicode"
 )
 
 type PackagesAssertions []PackageAssert