WIP

pkg/installer/client/local.go

@@ -0,0 +1,66 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/helpers"
+	"github.com/mudler/luet/pkg/installer"
+)
+
+type LocalClient struct {
+	Repository installer.Repository
+}
+
+func NewLocalClient(r installer.Repository) installer.Client {
+	return &LocalClient{Repository: r}
+}
+
+func (c *LocalClient) GetRepository() installer.Repository {
+	return c.Repository
+}
+
+func (c *LocalClient) SetRepository(r installer.Repository) {
+	c.Repository = r
+}
+func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
+
+	file, err := ioutil.TempFile(os.TempDir(), "localclient")
+	if err != nil {
+		return "", err
+	}
+	//defer os.Remove(file.Name())
+
+	err = helpers.CopyFile(filepath.Join(repo.GetUri(), artifact.GetPath()), file.Name())
+
+	return compiler.NewPackageArtifact(file.Name()), nil
+}
+func (c *LocalClient) DownloadFile(name string) (string, error) {
+
+	file, err := ioutil.TempFile(os.TempDir(), "localclient")
+	if err != nil {
+		return "", err
+	}
+	//defer os.Remove(file.Name())
+
+	err = helpers.CopyFile(filepath.Join(r.GetUri(), name), file.Name())
+
+	return file.Name(), err
+}

pkg/installer/installer.go

@@ -0,0 +1,184 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"sort"
+	"sync"
+
+	compiler "github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/pkg/errors"
+)
+
+type LuetInstaller struct {
+	PackageClient       Client
+	PackageRepositories Repositories
+	Concurrency         int
+}
+
+type ArtifactMatch struct {
+	Package    pkg.Package
+	Artifact   compiler.Artifact
+	Repository Repository
+}
+
+func NewLuetInstaller(concurrency int) Installer {
+	return &LuetInstaller{Concurrency: concurrency}
+}
+
+func (l *LuetInstaller) Install(p []pkg.Package, s *System) error {
+	// First get metas from all repos (and decodes trees)
+
+	Spinner(32)
+	defer SpinnerStop()
+	syncedRepos := Repositories{}
+	for _, r := range l.PackageRepositories {
+		repo, err := r.Sync(l.PackageClient)
+		if err != nil {
+			return errors.Wrap(err, "Failed syncing repository"+r.GetName())
+		}
+		syncedRepos = append(syncedRepos, repo)
+	}
+
+	// compute what to install and from where
+	sort.Sort(syncedRepos)
+
+	// First match packages against repositories by priority
+	matches := syncedRepos.PackageMatches(p)
+
+	// Get installed definition
+	installed, err := s.World()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating installed world ")
+	}
+
+	// compute a "big" world
+	allrepoWorld := syncedRepos.World()
+
+	// If installed exists in the world, we need to use them to make the solver point to them
+	realInstalled := []pkg.Package{}
+	for _, i := range installed {
+		var found pkg.Package
+	W:
+		for _, p := range allrepoWorld {
+			if p.Matches(i) {
+				found = p
+				break W
+			}
+		}
+
+		if found != nil {
+			realInstalled = append(realInstalled, found)
+
+		} else {
+			realInstalled = append(realInstalled, i)
+		}
+	}
+
+	allwanted := []pkg.Package{}
+	for _, wanted := range p {
+		var found pkg.Package
+
+	W:
+		for _, p := range allrepoWorld {
+			if p.Matches(wanted) {
+				found = p
+				break W
+			}
+		}
+
+		if found != nil {
+			allwanted = append(allwanted, found)
+
+		} else {
+			return errors.New("Package requested to install not found")
+		}
+	}
+
+	s := solver.NewSolver(realInstalled, allrepoWorld, pkg.NewInMemoryDatabase(false))
+	solution, err := s.Install(allwanted)
+
+	// Gathers things to install
+	toInstall := []ArtifactMatch{}
+	for _, assertion := range solution {
+		if assertion.Value && assertion.Package.IsFlagged() {
+			matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package})
+			if len(matches) != 1 {
+				return errors.New("Failed matching solutions against repository - where are definitions coming from?!")
+			}
+		W:
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+					toInstall = append(toInstall, ArtifactMatch{Package: assertion.Package, Artifact: artefact, Repository: matches[0].Repo})
+					break W
+				}
+			}
+		}
+	}
+
+	all := make(chan ArtifactMatch)
+
+	var wg = new(sync.WaitGroup)
+	for i := 0; i < l.Concurrency; i++ {
+		wg.Add(1)
+		go t.installerWorker(i, wg, all, s)
+	}
+
+	for _, c := range toInstall {
+		all <- c
+	}
+	close(all)
+	wg.Wait()
+
+	// Next: Look up for each solution with PackageMatches to check where to install.
+
+	// install (parallel)
+	// finalizers(sequential -  generate an ordered list of packagematches of installs from packagematches order for each package. Just make sure to run the finalizers ones.)
+	// mark the installation to the system db, along with the files that belongs to the package
+	return nil
+
+}
+
+func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error {
+
+}
+
+func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error {
+	defer wg.Done()
+
+	for p := range c {
+		err := l.installPackage(p, s)
+		if err != nil {
+			//TODO: Uninstall, rollback.
+			Fatal("Failed installing package" + p.Package.GetName())
+			return err
+		}
+	}
+
+	return nil
+}
+func (l *LuetInstaller) Uninstall(p []pkg.Package, s *System) error {
+	// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
+	return nil
+
+}
+
+func (l *LuetInstaller) Client(c Client)             { l.PackageClient = c }
+func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r }

pkg/installer/installer_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestInstaller(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Installer Suite")
+}

pkg/installer/interface.go

@@ -0,0 +1,54 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	compiler "github.com/mudler/luet/pkg/compiler"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	//"github.com/mudler/luet/pkg/solver"
+)
+
+type Installer interface {
+	Install([]pkg.Package, *System) error
+	Uninstall([]pkg.Package, *System) error
+	Client(Client)
+	Repositories([]Repository)
+}
+
+type Client interface {
+	DownloadArtifact(compiler.Artifact) (compiler.Artifact, error)
+	DownloadFile(string) (string, error)
+
+	GetRepository() Repository
+	SetRepository(Repository)
+}
+
+type Repositories []Repository
+
+type Repository interface {
+	GetName() string
+	GetUri() string
+	GetPriority() int
+	GetIndex() compiler.ArtifactIndex
+	GetTree() tree.Builder
+	Write(path string) error
+	Sync(Client) (Repository, error)
+	GetTreePath() string
+	SetTreePath(string)
+	GetType() string
+	SetType(string)
+}

pkg/installer/repository.go

@@ -0,0 +1,265 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/ghodss/yaml"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+	tree "github.com/mudler/luet/pkg/tree"
+	"github.com/pkg/errors"
+)
+
+type LuetRepository struct {
+	Name     string                 `json:"name"`
+	Uri      string                 `json:"uri"`
+	Priority int                    `json:"priority"`
+	Index    compiler.ArtifactIndex `json:"index"`
+	Tree     tree.Builder           `json:"-"`
+	TreePath string                 `json:"-"`
+	Type     string                 `json:"type"`
+}
+
+func GenerateRepository(name, uri string, priority int, src, tree string, db pkg.PackageDatabase) (Repository, error) {
+
+	art, err := buildPackageIndex(src)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewLuetRepository(name, uri, priority, art, db), nil
+}
+
+func NewLuetRepository(name, uri string, priority int, art []compiler.Artifact, db pkg.PackageDatabase) Repository {
+	return &LuetRepository{Index: art, Tree: tree.NewInstallerRecipe(db)}
+}
+
+func NewLuetRepositoryFromYaml(data []byte) (Repository, error) {
+	var p LuetRepository
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return &p, err
+	}
+	return &p, err
+}
+
+func buildPackageIndex(path string) ([]compiler.Artifact, error) {
+
+	var art []compiler.Artifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		artifact, err := compiler.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+		art = append(art, artifact)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
+func (r *LuetRepository) GetName() string {
+	return r.Name
+}
+func (r *LuetRepository) GetTreePath() string {
+	return r.TreePath
+}
+func (r *LuetRepository) SetTreePath(p string) {
+	r.TreePath = p
+}
+
+func (r *LuetRepository) GetType() string {
+	return r.Type
+}
+func (r *LuetRepository) SetType(p string) {
+	r.Type = p
+}
+
+func (r *LuetRepository) GetUri() string {
+	return r.Uri
+}
+func (r *LuetRepository) GetPriority() int {
+	return r.Priority
+}
+func (r *LuetRepository) GetIndex() compiler.ArtifactIndex {
+	return r.Index
+}
+func (r *LuetRepository) GetTree() tree.Builder {
+	return r.Tree
+}
+
+func (r *LuetRepository) Write(dst string) error {
+
+	os.MkdirAll(dst, os.ModePerm)
+	data, err := yaml.Marshal(r)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(filepath.Join(dst, "repository.yaml"), data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+	err = r.GetTree().Save(archive)
+	if err != nil {
+		return errors.Wrap(err, "Error met while saving the tree")
+	}
+	err = helpers.Tar(archive, filepath.Join(dst, "tree.tar"))
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+	return nil
+}
+
+func (r *LuetRepository) Sync(c Client) (Repository, error) {
+	c.SetRepository(r)
+
+	file, err := c.DownloadFile("repository.yaml")
+	if err != nil {
+		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+	}
+	dat, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading file "+file)
+	}
+	defer os.Remove(file)
+
+	repo, err := NewLuetRepositoryFromYaml(dat)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading repository from file "+file)
+
+	}
+
+	archivetree, err := c.DownloadFile("tree.tar")
+	if err != nil {
+		return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri())
+	}
+	defer os.RemoveAll(archivetree) // clean up
+
+	treefs, err := ioutil.TempDir(os.TempDir(), "treefs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	//defer os.RemoveAll(treefs) // clean up
+
+	// TODO: Following as option if archive as output?
+	// archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	// if err != nil {
+	// 	return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	// }
+	// defer os.RemoveAll(archive) // clean up
+
+	err = helpers.Untar(archivetree, treefs, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+	}
+
+	reciper := tree.NewInstallerRecipe(r.GetTree().Tree().GetPackageSet())
+	err = reciper.Load(treefs)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+	}
+	repo.SetTreePath(treefs)
+
+	return repo, nil
+}
+
+// TODO:
+
+func (r Repositories) Len() int      { return len(r) }
+func (r Repositories) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r Repositories) Less(i, j int) bool {
+	return r[i].GetPriority() < r[j].GetPriority()
+}
+
+func (r Repositories) World() []pkg.Package {
+	cache := map[string]pkg.Package{}
+	world := []pkg.Package{}
+
+	// Get Uniques. Walk in reverse so the definitions of most prio-repo overwrites lower ones
+	// In this way, when we will walk again later the deps sorting them by most higher prio we have better chance of success.
+	for i := len(r) - 1; i >= 0; i-- {
+		w, err := r[i].GetTree().Tree().World()
+		if err != nil {
+			Warning("Failed computing world for " + r[i].GetName())
+			continue
+		}
+		for _, p := range w {
+			cache[p.GetFingerPrint()] = p
+		}
+	}
+
+	for _, v := range cache {
+		world = append(world, v)
+	}
+
+	return world
+}
+
+type PackageMatch struct {
+	Repo    Repository
+	Package pkg.Package
+}
+
+func (re Repositories) PackageMatches(p []pkg.Package) []PackageMatch {
+	// TODO: Better heuristic. here we pick the first repo that contains the atom, sorted by priority but
+	// we should do a permutations and get the best match, and in case there are more solutions the user should be able to pick
+	sort.Sort(re)
+
+	var matches []PackageMatch
+PACKAGE:
+	for _, pack := range p {
+		for _, r := range re {
+			c, err := r.GetTree().Tree().GetPackageSet().FindPackage(pack)
+			if err == nil {
+				matches = append(matches, PackageMatch{Package: c, Repo: r})
+				continue PACKAGE
+			}
+		}
+	}
+
+	return matches
+
+}

pkg/installer/repository_test.go

@@ -0,0 +1,83 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer_test
+
+import (
+	"io/ioutil"
+	"os"
+
+	//	. "github.com/mudler/luet/pkg/installer"
+
+	"github.com/mudler/luet/pkg/helpers"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Repository", func() {
+	Context("Writes a repository definition", func() {
+		It("Writes a repo", func() {
+			//repo:=NewLuetRepository()
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
+
+			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
+			err = compiler.Prepare(1)
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+
+			spec.SetOutputPath(tmpdir)
+			artifact, err := compiler.Compile(2, false, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+			Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("test5"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("test6"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("artifact5\n"))
+			Expect(content2).To(Equal("artifact6\n"))
+
+		})
+
+	})
+})

pkg/installer/system.go

@@ -0,0 +1,17 @@
+package installer
+
+import (
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+)
+
+type System struct {
+	Database pkg.PackageDatabase
+	Target   string
+}
+
+func (s *System) World() ([]pkg.Package, error) {
+	t := tree.NewDefaultTree()
+	t.SetPackageSet(s.Database)
+	return t.World()
+}