Add CLI helpers and BuildTree heuristic

The BuildTree implementation was in parallel-build-tools, which actually makes
sense to share in the compiler as can be used as another way to trigger
builds.

The CLI helpers were split in several CIs
implementations. This moves a new layout where we expose several
packages, as we will refactor things slowly.

pkg/api/client/client_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestClient(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Client API Suite")
+}

pkg/api/client/search.go

@@ -0,0 +1,131 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/api/client/utils"
+)
+
+func TreePackages(treedir string) (searchResult SearchResult, err error) {
+	var res []byte
+	res, err = utils.RunSHOUT("tree", fmt.Sprintf("luet tree pkglist --tree %s --output json", treedir))
+	if err != nil {
+		fmt.Println(string(res))
+		return
+	}
+	json.Unmarshal(res, &searchResult)
+	return
+}
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+type SearchResult struct {
+	Packages []Package
+}
+
+type Package struct {
+	Name, Category, Version, Path string
+}
+
+func (p Package) String() string {
+	return fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version)
+}
+
+func (p Package) Image(repository string) string {
+	return fmt.Sprintf("%s:%s-%s-%s", repository, p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageTag() string {
+	// ${name}-${category}-${version//+/-}
+	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageMetadata(repository string) string {
+	return fmt.Sprintf("%s.metadata.yaml", p.Image(repository))
+}
+
+func (p Package) ImageAvailable(repository string) bool {
+	return imageAvailable(p.Image(repository))
+}
+
+func (p Package) Equal(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category && p.Version == pp.Version {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualS(s string) bool {
+	if s == fmt.Sprintf("%s/%s", p.Category, p.Name) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualSV(s string) bool {
+	if s == fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualNoV(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category {
+		return true
+	}
+	return false
+}
+
+func (s SearchResult) FilterByCategory(cat string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if r.Category == cat {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+func (s SearchResult) FilterByName(name string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if !strings.Contains(r.Name, name) {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+type Packages []Package
+
+func (p Packages) Exist(pp Package) bool {
+	for _, pi := range p {
+		if pp.Equal(pi) {
+			return true
+		}
+	}
+	return false
+}

pkg/api/client/search_test.go

@@ -0,0 +1,47 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	. "github.com/mudler/luet/pkg/api/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Client CLI API", func() {
+	Context("Reads a package tree from the luet CLI", func() {
+		It("Correctly detect packages", func() {
+			t, err := TreePackages("../../../tests/fixtures/alpine")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t).ToNot(BeNil())
+			Expect(len(t.Packages)).To(Equal(1))
+			Expect(t.Packages[0].Name).To(Equal("alpine"))
+			Expect(t.Packages[0].Category).To(Equal("seed"))
+			Expect(t.Packages[0].Version).To(Equal("1.0"))
+			Expect(t.Packages[0].ImageAvailable("foo")).To(BeFalse())
+			Expect(t.Packages[0].Equal(t.Packages[0])).To(BeTrue())
+			Expect(t.Packages[0].Equal(Package{})).To(BeFalse())
+			Expect(t.Packages[0].EqualNoV(Package{Name: "alpine", Category: "seed"})).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpine")).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpinev")).To(BeFalse())
+			Expect(t.Packages[0].EqualSV("seed/alpine@1.0")).To(BeTrue())
+			Expect(t.Packages[0].Image("foo")).To(Equal("foo:alpine-seed-1.0"))
+			Expect(Packages(t.Packages).Exist(t.Packages[0])).To(BeTrue())
+			Expect(Packages(t.Packages).Exist(Package{})).To(BeFalse())
+
+		})
+	})
+})

pkg/api/client/utils/sh.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func RunSHOUT(stepName, bashFragment string) ([]byte, error) {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+
+	cmd.Env = os.Environ()
+	//	log.Printf("Running in background: %v", stepName)
+
+	return cmd.CombinedOutput()
+}
+
+func RunSH(stepName, bashFragment string) error {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+	log.Printf("Running: %v (%v)", stepName, bashFragment)
+
+	return cmd.Run()
+}
+
+func bashWrap(cmd string) string {
+	return `
+set -o errexit
+set -o nounset
+` + cmd + `
+`
+}

pkg/compiler/buildtree.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+func rankMapStringInt(values map[string]int) []string {
+	type kv struct {
+		Key   string
+		Value int
+	}
+	var ss []kv
+	for k, v := range values {
+		ss = append(ss, kv{k, v})
+	}
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Value > ss[j].Value
+	})
+	ranked := make([]string, len(values))
+	for i, kv := range ss {
+		ranked[i] = kv.Key
+	}
+	return ranked
+}
+
+type BuildTree struct {
+	order map[string]int
+}
+
+func (bt *BuildTree) Increase(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	if _, ok := bt.order[s]; !ok {
+		bt.order[s] = 0
+	}
+
+	bt.order[s]++
+}
+
+func (bt *BuildTree) Reset(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	bt.order[s] = 0
+}
+
+func (bt *BuildTree) Level(s string) int {
+	return bt.order[s]
+}
+
+func ints(input []int) []int {
+	u := make([]int, 0, len(input))
+	m := make(map[int]bool)
+
+	for _, val := range input {
+		if _, ok := m[val]; !ok {
+			m[val] = true
+			u = append(u, val)
+		}
+	}
+
+	return u
+}
+
+func (bt *BuildTree) AllInLevel(l int) []string {
+	var all []string
+	for k, v := range bt.order {
+		if v == l {
+			all = append(all, k)
+		}
+	}
+	return all
+}
+
+func (bt *BuildTree) Order(compilationTree map[string]map[string]interface{}) {
+	sentinel := false
+	for !sentinel {
+		sentinel = true
+
+	LEVEL:
+		for _, l := range bt.AllLevels() {
+
+			for _, j := range bt.AllInLevel(l) {
+				for _, k := range bt.AllInLevel(l) {
+					if j == k {
+						continue
+					}
+					if _, ok := compilationTree[j][k]; ok {
+						if bt.Level(j) == bt.Level(k) {
+							bt.Increase(j)
+							sentinel = false
+							break LEVEL
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (bt *BuildTree) AllLevels() []int {
+	var all []int
+	for _, v := range bt.order {
+		all = append(all, v)
+	}
+
+	sort.Sort(sort.IntSlice(all))
+
+	return ints(all)
+}
+
+func (bt *BuildTree) JSON() (string, error) {
+	type buildjob struct {
+		Jobs []string `json:"packages"`
+	}
+
+	result := []buildjob{}
+	for _, l := range bt.AllLevels() {
+		result = append(result, buildjob{Jobs: bt.AllInLevel(l)})
+	}
+	dat, err := json.Marshal(&result)
+	return string(dat), err
+}

pkg/compiler/compiler.go

@@ -704,6 +704,51 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (sol
 	return dependencies, nil
 }
 
+// BuildTree returns a BuildTree which represent the order in which specs should be compiled.
+// It places specs into levels, and each level can be built in parallel. The root nodes starting from the top.
+// A BuildTree can be marshaled into JSON or walked like:
+// for _, l := range bt.AllLevels() {
+//	fmt.Println(strings.Join(bt.AllInLevel(l), " "))
+// }
+func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspecs) (*BuildTree, error) {
+	compilationTree := map[string]map[string]interface{}{}
+	bt := &BuildTree{}
+
+	for _, sp := range compilerSpecs.All() {
+		ass, err := cs.ComputeDepTree(sp)
+		if err != nil {
+			return nil, err
+		}
+		bt.Reset(fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName()))
+		for _, p := range ass {
+			bt.Reset(fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName()))
+
+			spec, err := cs.FromPackage(p.Package)
+			if err != nil {
+				return nil, err
+			}
+			ass, err := cs.ComputeDepTree(spec)
+			if err != nil {
+				return nil, err
+			}
+			for _, r := range ass {
+				if compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] == nil {
+					compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = make(map[string]interface{})
+				}
+				compilationTree[fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())][fmt.Sprintf("%s/%s", r.Package.GetCategory(), r.Package.GetName())] = nil
+			}
+			if compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] == nil {
+				compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())] = make(map[string]interface{})
+			}
+			compilationTree[fmt.Sprintf("%s/%s", sp.GetPackage().GetCategory(), sp.GetPackage().GetName())][fmt.Sprintf("%s/%s", p.Package.GetCategory(), p.Package.GetName())] = nil
+		}
+	}
+
+	bt.Order(compilationTree)
+
+	return bt, nil
+}
+
 // ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
 func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
 	// Generate a set with all the deps of the provided specs

pkg/compiler/compiler_test.go

@@ -660,6 +660,30 @@ var _ = Describe("Compiler", func() {
 			Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
 		})
 
+		It("Generates a correct buildtree", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/complex/selection")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
+			Expect(err).ToNot(HaveOccurred())
+
+			bt, err := compiler.BuildTree(compilerspec.LuetCompilationspecs{*spec})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bt.AllLevels()).To(Equal([]int{0, 1, 2, 3, 4, 5}))
+			Expect(bt.AllInLevel(0)).To(Equal([]string{"layer/build"}))
+			Expect(bt.AllInLevel(1)).To(Equal([]string{"layer/sabayon-build-portage"}))
+			Expect(bt.AllInLevel(2)).To(Equal([]string{"layer/build-sabayon-overlay"}))
+			Expect(bt.AllInLevel(3)).To(Equal([]string{"layer/build-sabayon-overlays"}))
+			Expect(bt.AllInLevel(4)).To(ContainElements("sys-kernel/linux-sabayon", "sys-kernel/sabayon-sources"))
+			Expect(bt.AllInLevel(5)).To(Equal([]string{"sys-fs-5.4.2/vhba"}))
+		})
+
 		It("Compiles complex dependencies trees with best matches", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "complex")