Add CLI helpers and BuildTree heuristic

The BuildTree implementation was in parallel-build-tools, which actually makes
sense to share in the compiler as can be used as another way to trigger
builds.

The CLI helpers were split in several CIs
implementations. This moves a new layout where we expose several
packages, as we will refactor things slowly.

pkg/compiler/buildtree.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+func rankMapStringInt(values map[string]int) []string {
+	type kv struct {
+		Key   string
+		Value int
+	}
+	var ss []kv
+	for k, v := range values {
+		ss = append(ss, kv{k, v})
+	}
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Value > ss[j].Value
+	})
+	ranked := make([]string, len(values))
+	for i, kv := range ss {
+		ranked[i] = kv.Key
+	}
+	return ranked
+}
+
+type BuildTree struct {
+	order map[string]int
+}
+
+func (bt *BuildTree) Increase(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	if _, ok := bt.order[s]; !ok {
+		bt.order[s] = 0
+	}
+
+	bt.order[s]++
+}
+
+func (bt *BuildTree) Reset(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	bt.order[s] = 0
+}
+
+func (bt *BuildTree) Level(s string) int {
+	return bt.order[s]
+}
+
+func ints(input []int) []int {
+	u := make([]int, 0, len(input))
+	m := make(map[int]bool)
+
+	for _, val := range input {
+		if _, ok := m[val]; !ok {
+			m[val] = true
+			u = append(u, val)
+		}
+	}
+
+	return u
+}
+
+func (bt *BuildTree) AllInLevel(l int) []string {
+	var all []string
+	for k, v := range bt.order {
+		if v == l {
+			all = append(all, k)
+		}
+	}
+	return all
+}
+
+func (bt *BuildTree) Order(compilationTree map[string]map[string]interface{}) {
+	sentinel := false
+	for !sentinel {
+		sentinel = true
+
+	LEVEL:
+		for _, l := range bt.AllLevels() {
+
+			for _, j := range bt.AllInLevel(l) {
+				for _, k := range bt.AllInLevel(l) {
+					if j == k {
+						continue
+					}
+					if _, ok := compilationTree[j][k]; ok {
+						if bt.Level(j) == bt.Level(k) {
+							bt.Increase(j)
+							sentinel = false
+							break LEVEL
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (bt *BuildTree) AllLevels() []int {
+	var all []int
+	for _, v := range bt.order {
+		all = append(all, v)
+	}
+
+	sort.Sort(sort.IntSlice(all))
+
+	return ints(all)
+}
+
+func (bt *BuildTree) JSON() (string, error) {
+	type buildjob struct {
+		Jobs []string `json:"packages"`
+	}
+
+	result := []buildjob{}
+	for _, l := range bt.AllLevels() {
+		result = append(result, buildjob{Jobs: bt.AllInLevel(l)})
+	}
+	dat, err := json.Marshal(&result)
+	return string(dat), err
+}

pkg/compiler/compiler.go

@@ -704,6 +704,51 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
 	return dependencies, nil
 }
 
+// BuildTree returns a BuildTree which represent the order in which specs should be compiled.
+// It places specs into levels, and each level can be built in parallel. The root nodes starting from the top.
+// A BuildTree can be marshaled into JSON or walked like:
+// for _, l := range bt.AllLevels() {
+//	fmt.Println(strings.Join(bt.AllInLevel(l), " "))
+// }
+func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspecs) (*BuildTree, error) {
+	compilationTree := map[string]map[string]interface{}{}
+	bt := &BuildTree{}
+
+	for _, sp := range compilerSpecs.All() {
+		ass, err := cs.ComputeDepTree(sp)
+		if err != nil {
+			return nil, err
+		}
+		bt.Reset(fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName()))
+		for _, p := range ass {
+			bt.Reset(fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName()))
+
+			spec, err := cs.FromPackage(p.Package)
+			if err != nil {
+				return nil, err
+			}
+			ass, err := cs.ComputeDepTree(spec)
+			if err != nil {
+				return nil, err
+			}
+			for _, r := range ass {
+				if compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] == nil {
+					compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = make(map[string]interface{})
+				}
+				compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())][fmt.Sprintf("%s/%s", r.Package.GetCategory(), r.Package.GetName())] = nil
+			}
+			if compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] == nil {
+				compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] = make(map[string]interface{})
+			}
+			compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())][fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = nil
+		}
+	}
+
+	bt.Order(compilationTree)
+
+	return bt, nil
+}
+
 // ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
 func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
 	// Generate a set with all the deps of the provided specs

pkg/compiler/compiler_test.go

@@ -660,6 +660,30 @@ var _ = Describe("Compiler", func() {
 			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Generates a correct buildtree", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/complex/selection")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
+			Expect(err).ToNot(HaveOccurred())
+
+			bt, err := compiler.BuildTree(compilerspec.LuetCompilationspecs{*spec})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bt.AllLevels()).To(Equal([]int{0, 1, 2, 3, 4, 5}))
+			Expect(bt.AllInLevel(0)).To(Equal([]string{"layer/build"}))
+			Expect(bt.AllInLevel(1)).To(Equal([]string{"layer/sabayon-build-portage"}))
+			Expect(bt.AllInLevel(2)).To(Equal([]string{"layer/build-sabayon-overlay"}))
+			Expect(bt.AllInLevel(3)).To(Equal([]string{"layer/build-sabayon-overlays"}))
+			Expect(bt.AllInLevel(4)).To(ContainElements("sys-kernel/linux-sabayon", "sys-kernel/sabayon-sources"))
+			Expect(bt.AllInLevel(5)).To(Equal([]string{"sys-fs-5.4.2/vhba"}))
+		})
+
 		It("Compiles complex dependencies trees with best matches", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "complex")