From a2f23d3bf567853c42f14c8a05517dd666d65b36 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 22 Nov 2019 21:01:38 +0100 Subject: [PATCH] WIP --- pkg/installer/client/local.go | 66 +++++++ pkg/installer/installer.go | 184 ++++++++++++++++++ pkg/installer/installer_suite_test.go | 28 +++ pkg/installer/interface.go | 54 ++++++ pkg/installer/repository.go | 265 ++++++++++++++++++++++++++ pkg/installer/repository_test.go | 83 ++++++++ pkg/installer/system.go | 17 ++ 7 files changed, 697 insertions(+) create mode 100644 pkg/installer/client/local.go create mode 100644 pkg/installer/installer.go create mode 100644 pkg/installer/installer_suite_test.go create mode 100644 pkg/installer/interface.go create mode 100644 pkg/installer/repository.go create mode 100644 pkg/installer/repository_test.go create mode 100644 pkg/installer/system.go diff --git a/pkg/installer/client/local.go b/pkg/installer/client/local.go new file mode 100644 index 00000000..be2fa4b8 --- /dev/null +++ b/pkg/installer/client/local.go @@ -0,0 +1,66 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package client + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/helpers" + "github.com/mudler/luet/pkg/installer" +) + +type LocalClient struct { + Repository installer.Repository +} + +func NewLocalClient(r installer.Repository) installer.Client { + return &LocalClient{Repository: r} +} + +func (c *LocalClient) GetRepository() installer.Repository { + return c.Repository +} + +func (c *LocalClient) SetRepository(r installer.Repository) { + c.Repository = r +} +func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { + + file, err := ioutil.TempFile(os.TempDir(), "localclient") + if err != nil { + return "", err + } + //defer os.Remove(file.Name()) + + err = helpers.CopyFile(filepath.Join(repo.GetUri(), artifact.GetPath()), file.Name()) + + return compiler.NewPackageArtifact(file.Name()), nil +} +func (c *LocalClient) DownloadFile(name string) (string, error) { + + file, err := ioutil.TempFile(os.TempDir(), "localclient") + if err != nil { + return "", err + } + //defer os.Remove(file.Name()) + + err = helpers.CopyFile(filepath.Join(r.GetUri(), name), file.Name()) + + return file.Name(), err +} diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go new file mode 100644 index 00000000..f75d245d --- /dev/null +++ b/pkg/installer/installer.go @@ -0,0 +1,184 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer + +import ( + "sort" + "sync" + + compiler "github.com/mudler/luet/pkg/compiler" + . "github.com/mudler/luet/pkg/logger" + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/solver" + + "github.com/pkg/errors" +) + +type LuetInstaller struct { + PackageClient Client + PackageRepositories Repositories + Concurrency int +} + +type ArtifactMatch struct { + Package pkg.Package + Artifact compiler.Artifact + Repository Repository +} + +func NewLuetInstaller(concurrency int) Installer { + return &LuetInstaller{Concurrency: concurrency} +} + +func (l *LuetInstaller) Install(p []pkg.Package, s *System) error { + // First get metas from all repos (and decodes trees) + + Spinner(32) + defer SpinnerStop() + syncedRepos := Repositories{} + for _, r := range l.PackageRepositories { + repo, err := r.Sync(l.PackageClient) + if err != nil { + return errors.Wrap(err, "Failed syncing repository"+r.GetName()) + } + syncedRepos = append(syncedRepos, repo) + } + + // compute what to install and from where + sort.Sort(syncedRepos) + + // First match packages against repositories by priority + matches := syncedRepos.PackageMatches(p) + + // Get installed definition + installed, err := s.World() + if err != nil { + return errors.Wrap(err, "Failed generating installed world ") + } + + // compute a "big" world + allrepoWorld := syncedRepos.World() + + // If installed exists in the world, we need to use them to make the solver point to them + realInstalled := []pkg.Package{} + for _, i := range installed { + var found pkg.Package + W: + for _, p := range allrepoWorld { + if p.Matches(i) { + found = p + break W + } + } + + if found != nil { + realInstalled = append(realInstalled, found) + + } else { + realInstalled = append(realInstalled, i) + } + } + + allwanted := []pkg.Package{} + for _, wanted := range p { + var found pkg.Package + + W: + for _, p := range allrepoWorld { + if p.Matches(wanted) { + found = p + break W + } + } + + if found != nil { + allwanted = append(allwanted, found) + + } else { + return errors.New("Package requested to install not found") + } + } + + s := solver.NewSolver(realInstalled, allrepoWorld, pkg.NewInMemoryDatabase(false)) + solution, err := s.Install(allwanted) + + // Gathers things to install + toInstall := []ArtifactMatch{} + for _, assertion := range solution { + if assertion.Value && assertion.Package.IsFlagged() { + matches := syncedRepos.PackageMatches([]pkg.Package{assertion.Package}) + if len(matches) != 1 { + return errors.New("Failed matching solutions against repository - where are definitions coming from?!") + } + W: + for _, artefact := range matches[0].Repo.GetIndex() { + if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) { + toInstall = append(toInstall, ArtifactMatch{Package: assertion.Package, Artifact: artefact, Repository: matches[0].Repo}) + break W + } + } + } + } + + all := make(chan ArtifactMatch) + + var wg = new(sync.WaitGroup) + for i := 0; i < l.Concurrency; i++ { + wg.Add(1) + go t.installerWorker(i, wg, all, s) + } + + for _, c := range toInstall { + all <- c + } + close(all) + wg.Wait() + + // Next: Look up for each solution with PackageMatches to check where to install. + + // install (parallel) + // finalizers(sequential - generate an ordered list of packagematches of installs from packagematches order for each package. Just make sure to run the finalizers ones.) + // mark the installation to the system db, along with the files that belongs to the package + return nil + +} + +func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error { + +} + +func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch, s *System) error { + defer wg.Done() + + for p := range c { + err := l.installPackage(p, s) + if err != nil { + //TODO: Uninstall, rollback. + Fatal("Failed installing package" + p.Package.GetName()) + return err + } + } + + return nil +} +func (l *LuetInstaller) Uninstall(p []pkg.Package, s *System) error { + // compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db + return nil + +} + +func (l *LuetInstaller) Client(c Client) { l.PackageClient = c } +func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r } diff --git a/pkg/installer/installer_suite_test.go b/pkg/installer/installer_suite_test.go new file mode 100644 index 00000000..e14aee8c --- /dev/null +++ b/pkg/installer/installer_suite_test.go @@ -0,0 +1,28 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestInstaller(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Installer Suite") +} diff --git a/pkg/installer/interface.go b/pkg/installer/interface.go new file mode 100644 index 00000000..53c36acb --- /dev/null +++ b/pkg/installer/interface.go @@ -0,0 +1,54 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer + +import ( + compiler "github.com/mudler/luet/pkg/compiler" + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/tree" + //"github.com/mudler/luet/pkg/solver" +) + +type Installer interface { + Install([]pkg.Package, *System) error + Uninstall([]pkg.Package, *System) error + Client(Client) + Repositories([]Repository) +} + +type Client interface { + DownloadArtifact(compiler.Artifact) (compiler.Artifact, error) + DownloadFile(string) (string, error) + + GetRepository() Repository + SetRepository(Repository) +} + +type Repositories []Repository + +type Repository interface { + GetName() string + GetUri() string + GetPriority() int + GetIndex() compiler.ArtifactIndex + GetTree() tree.Builder + Write(path string) error + Sync(Client) (Repository, error) + GetTreePath() string + SetTreePath(string) + GetType() string + SetType(string) +} diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go new file mode 100644 index 00000000..0d988b03 --- /dev/null +++ b/pkg/installer/repository.go @@ -0,0 +1,265 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer + +import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + . "github.com/mudler/luet/pkg/logger" + + "github.com/ghodss/yaml" + "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/helpers" + pkg "github.com/mudler/luet/pkg/package" + tree "github.com/mudler/luet/pkg/tree" + "github.com/pkg/errors" +) + +type LuetRepository struct { + Name string `json:"name"` + Uri string `json:"uri"` + Priority int `json:"priority"` + Index compiler.ArtifactIndex `json:"index"` + Tree tree.Builder `json:"-"` + TreePath string `json:"-"` + Type string `json:"type"` +} + +func GenerateRepository(name, uri string, priority int, src, tree string, db pkg.PackageDatabase) (Repository, error) { + + art, err := buildPackageIndex(src) + if err != nil { + return nil, err + } + + return NewLuetRepository(name, uri, priority, art, db), nil +} + +func NewLuetRepository(name, uri string, priority int, art []compiler.Artifact, db pkg.PackageDatabase) Repository { + return &LuetRepository{Index: art, Tree: tree.NewInstallerRecipe(db)} +} + +func NewLuetRepositoryFromYaml(data []byte) (Repository, error) { + var p LuetRepository + err := yaml.Unmarshal(data, &p) + if err != nil { + return &p, err + } + return &p, err +} + +func buildPackageIndex(path string) ([]compiler.Artifact, error) { + + var art []compiler.Artifact + var ff = func(currentpath string, info os.FileInfo, err error) error { + + if !strings.HasSuffix(info.Name(), ".metadata.yaml") { + return nil // Skip with no errors + } + + dat, err := ioutil.ReadFile(currentpath) + if err != nil { + return errors.Wrap(err, "Error reading file "+currentpath) + } + + artifact, err := compiler.NewPackageArtifactFromYaml(dat) + if err != nil { + return errors.Wrap(err, "Error reading yaml "+currentpath) + } + art = append(art, artifact) + + return nil + } + + err := filepath.Walk(path, ff) + if err != nil { + return nil, err + + } + return art, nil +} + +func (r *LuetRepository) GetName() string { + return r.Name +} +func (r *LuetRepository) GetTreePath() string { + return r.TreePath +} +func (r *LuetRepository) SetTreePath(p string) { + r.TreePath = p +} + +func (r *LuetRepository) GetType() string { + return r.Type +} +func (r *LuetRepository) SetType(p string) { + r.Type = p +} + +func (r *LuetRepository) GetUri() string { + return r.Uri +} +func (r *LuetRepository) GetPriority() int { + return r.Priority +} +func (r *LuetRepository) GetIndex() compiler.ArtifactIndex { + return r.Index +} +func (r *LuetRepository) GetTree() tree.Builder { + return r.Tree +} + +func (r *LuetRepository) Write(dst string) error { + + os.MkdirAll(dst, os.ModePerm) + data, err := yaml.Marshal(r) + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(dst, "repository.yaml"), data, os.ModePerm) + if err != nil { + return err + } + archive, err := ioutil.TempDir(os.TempDir(), "archive") + if err != nil { + return errors.Wrap(err, "Error met while creating tempdir for archive") + } + defer os.RemoveAll(archive) // clean up + err = r.GetTree().Save(archive) + if err != nil { + return errors.Wrap(err, "Error met while saving the tree") + } + err = helpers.Tar(archive, filepath.Join(dst, "tree.tar")) + if err != nil { + return errors.Wrap(err, "Error met while creating package archive") + } + return nil +} + +func (r *LuetRepository) Sync(c Client) (Repository, error) { + c.SetRepository(r) + + file, err := c.DownloadFile("repository.yaml") + if err != nil { + return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri()) + } + dat, err := ioutil.ReadFile(file) + if err != nil { + return nil, errors.Wrap(err, "Error reading file "+file) + } + defer os.Remove(file) + + repo, err := NewLuetRepositoryFromYaml(dat) + if err != nil { + return nil, errors.Wrap(err, "Error reading repository from file "+file) + + } + + archivetree, err := c.DownloadFile("tree.tar") + if err != nil { + return nil, errors.Wrap(err, "While downloading repository.yaml from "+r.GetUri()) + } + defer os.RemoveAll(archivetree) // clean up + + treefs, err := ioutil.TempDir(os.TempDir(), "treefs") + if err != nil { + return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") + } + //defer os.RemoveAll(treefs) // clean up + + // TODO: Following as option if archive as output? + // archive, err := ioutil.TempDir(os.TempDir(), "archive") + // if err != nil { + // return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") + // } + // defer os.RemoveAll(archive) // clean up + + err = helpers.Untar(archivetree, treefs, false) + if err != nil { + return nil, errors.Wrap(err, "Error met while unpacking rootfs") + } + + reciper := tree.NewInstallerRecipe(r.GetTree().Tree().GetPackageSet()) + err = reciper.Load(treefs) + if err != nil { + return nil, errors.Wrap(err, "Error met while unpacking rootfs") + } + repo.SetTreePath(treefs) + + return repo, nil +} + +// TODO: + +func (r Repositories) Len() int { return len(r) } +func (r Repositories) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r Repositories) Less(i, j int) bool { + return r[i].GetPriority() < r[j].GetPriority() +} + +func (r Repositories) World() []pkg.Package { + cache := map[string]pkg.Package{} + world := []pkg.Package{} + + // Get Uniques. Walk in reverse so the definitions of most prio-repo overwrites lower ones + // In this way, when we will walk again later the deps sorting them by most higher prio we have better chance of success. + for i := len(r) - 1; i >= 0; i-- { + w, err := r[i].GetTree().Tree().World() + if err != nil { + Warning("Failed computing world for " + r[i].GetName()) + continue + } + for _, p := range w { + cache[p.GetFingerPrint()] = p + } + } + + for _, v := range cache { + world = append(world, v) + } + + return world +} + +type PackageMatch struct { + Repo Repository + Package pkg.Package +} + +func (re Repositories) PackageMatches(p []pkg.Package) []PackageMatch { + // TODO: Better heuristic. here we pick the first repo that contains the atom, sorted by priority but + // we should do a permutations and get the best match, and in case there are more solutions the user should be able to pick + sort.Sort(re) + + var matches []PackageMatch +PACKAGE: + for _, pack := range p { + for _, r := range re { + c, err := r.GetTree().Tree().GetPackageSet().FindPackage(pack) + if err == nil { + matches = append(matches, PackageMatch{Package: c, Repo: r}) + continue PACKAGE + } + } + } + + return matches + +} diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go new file mode 100644 index 00000000..1797dcda --- /dev/null +++ b/pkg/installer/repository_test.go @@ -0,0 +1,83 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package installer_test + +import ( + "io/ioutil" + "os" + + // . "github.com/mudler/luet/pkg/installer" + + "github.com/mudler/luet/pkg/helpers" + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/tree" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Repository", func() { + Context("Writes a repository definition", func() { + It("Writes a repo", func() { + //repo:=NewLuetRepository() + + tmpdir, err := ioutil.TempDir("", "tree") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + + err := generalRecipe.Load("../../tests/fixtures/buildable") + Expect(err).ToNot(HaveOccurred()) + Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point + + Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(3)) + + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet()) + err = compiler.Prepare(1) + Expect(err).ToNot(HaveOccurred()) + + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + + Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) + + tmpdir, err := ioutil.TempDir("", "tree") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"})) + Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"})) + + spec.SetOutputPath(tmpdir) + artifact, err := compiler.Compile(2, false, spec) + Expect(err).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) + Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + + Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) + Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) + + content1, err := helpers.Read(spec.Rel("test5")) + Expect(err).ToNot(HaveOccurred()) + content2, err := helpers.Read(spec.Rel("test6")) + Expect(err).ToNot(HaveOccurred()) + Expect(content1).To(Equal("artifact5\n")) + Expect(content2).To(Equal("artifact6\n")) + + }) + + }) +}) diff --git a/pkg/installer/system.go b/pkg/installer/system.go new file mode 100644 index 00000000..6316c9c6 --- /dev/null +++ b/pkg/installer/system.go @@ -0,0 +1,17 @@ +package installer + +import ( + pkg "github.com/mudler/luet/pkg/package" + "github.com/mudler/luet/pkg/tree" +) + +type System struct { + Database pkg.PackageDatabase + Target string +} + +func (s *System) World() ([]pkg.Package, error) { + t := tree.NewDefaultTree() + t.SetPackageSet(s.Database) + return t.World() +}