From c7652c8a70251c0f412c53a693af0f6d375c10dc Mon Sep 17 00:00:00 2001 From: Daniele Rondina Date: Fri, 20 Mar 2020 23:47:35 +0100 Subject: [PATCH] Added `labels` to package definition * cmd/search: Add support for search of the packages with a specific label. * review Search method of Repositories for permit different search modes. * labels are k/v attributes and could be matched through label key (with HasLabel method) or through regex that use "$key" + "=" + "$value" --- cmd/search.go | 45 ++++++++++--- pkg/installer/repository.go | 44 +++++++++++-- pkg/package/database.go | 3 + pkg/package/database_boltdb.go | 59 +++++++++++++++++ pkg/package/database_mem.go | 60 ++++++++++++++++++ pkg/package/package.go | 63 +++++++++++++++++-- pkg/package/package_test.go | 30 +++++++++ pkg/tree/tree_test.go | 17 +++++ tests/fixtures/labels/pkgA/0.1/build.yaml | 3 + .../fixtures/labels/pkgA/0.1/definition.yaml | 6 ++ 10 files changed, 311 insertions(+), 19 deletions(-) create mode 100644 tests/fixtures/labels/pkgA/0.1/build.yaml create mode 100644 tests/fixtures/labels/pkgA/0.1/definition.yaml diff --git a/cmd/search.go b/cmd/search.go index 2e97aeab..6409c4d3 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -17,7 +17,6 @@ package cmd import ( "os" "path/filepath" - "regexp" . "github.com/mudler/luet/pkg/config" installer "github.com/mudler/luet/pkg/installer" @@ -51,6 +50,8 @@ var searchCmd = &cobra.Command{ discount := LuetCfg.Viper.GetFloat64("solver.discount") rate := LuetCfg.Viper.GetFloat64("solver.rate") attempts := LuetCfg.Viper.GetInt("solver.max_attempts") + searchWithLabel, _ := cmd.Flags().GetBool("by-label") + searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex") LuetCfg.GetSolverOptions().Type = stype LuetCfg.GetSolverOptions().LearnRate = float32(rate) @@ -70,8 +71,12 @@ var searchCmd = &cobra.Command{ repos = append(repos, r) } - inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()}) - + inst := installer.NewLuetInstaller( + installer.LuetInstallerOptions{ + Concurrency: LuetCfg.GetGeneral().Concurrency, + SolverOptions: *LuetCfg.GetSolverOptions(), + }, + ) inst.Repositories(repos) synced, err := inst.SyncRepositories(false) if err != nil { @@ -80,7 +85,14 @@ var searchCmd = &cobra.Command{ Info("--- Search results: ---") - matches := synced.Search(args[0]) + matches := []installer.PackageMatch{} + if searchWithLabel { + matches = synced.SearchLabel(args[0]) + } else if searchWithLabelMatch { + matches = synced.SearchLabelMatch(args[0]) + } else { + matches = synced.Search(args[0]) + } for _, m := range matches { Info(":package:", m.Package.GetCategory(), m.Package.GetName(), m.Package.GetVersion(), "repository:", m.Repo.GetName()) @@ -94,14 +106,25 @@ var searchCmd = &cobra.Command{ systemDB = pkg.NewInMemoryDatabase(true) } system := &installer.System{Database: systemDB, Target: LuetCfg.GetSystem().Rootfs} - var term = regexp.MustCompile(args[0]) - for _, k := range system.Database.GetPackages() { - pack, err := system.Database.GetPackage(k) - if err == nil && term.MatchString(pack.GetName()) { - Info(":package:", pack.GetCategory(), pack.GetName(), pack.GetVersion()) - } + var err error + iMatches := []pkg.Package{} + if searchWithLabel { + iMatches, err = system.Database.FindPackageLabel(args[0]) + } else if searchWithLabelMatch { + iMatches, err = system.Database.FindPackageLabelMatch(args[0]) + } else { + iMatches, err = system.Database.FindPackageMatch(args[0]) } + + if err != nil { + Fatal("Error: " + err.Error()) + } + + for _, pack := range iMatches { + Info(":package:", pack.GetCategory(), pack.GetName(), pack.GetVersion()) + } + } }, @@ -119,5 +142,7 @@ func init() { searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate") searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate") searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts") + searchCmd.Flags().Bool("by-label", false, "Search packages through label") + searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex") RootCmd.AddCommand(searchCmd) } diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 77e747f2..abe25364 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -21,7 +21,6 @@ import ( "os" "path" "path/filepath" - "regexp" "sort" "strconv" "strings" @@ -69,6 +68,19 @@ type LuetSystemRepositorySerialized struct { TreeChecksums compiler.Checksums `json:"treechecksums"` } +type LuetSearchModeType string + +const ( + SLabel LuetSearchModeType = "label" + SRegexPkg LuetSearchModeType = "regexPkg" + SRegexLabel LuetSearchModeType = "regexLabel" +) + +type LuetSearchOpts struct { + Pattern string + Mode LuetSearchModeType +} + func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) { art, err := buildPackageIndex(src) @@ -554,14 +566,24 @@ PACKAGE: } -func (re Repositories) Search(s string) []PackageMatch { +func (re Repositories) SearchPackages(p string, o LuetSearchOpts) []PackageMatch { sort.Sort(re) - var term = regexp.MustCompile(s) var matches []PackageMatch + var err error for _, r := range re { - for _, pack := range r.GetTree().GetDatabase().World() { - if term.MatchString(pack.GetName()) { + var repoMatches []pkg.Package + if o.Mode == SRegexPkg { + repoMatches, err = r.GetTree().GetDatabase().FindPackageMatch(p) + + } else if o.Mode == SLabel { + repoMatches, err = r.GetTree().GetDatabase().FindPackageLabel(p) + } else if o.Mode == SRegexLabel { + repoMatches, err = r.GetTree().GetDatabase().FindPackageLabelMatch(p) + } + + if err == nil && len(repoMatches) > 0 { + for _, pack := range repoMatches { matches = append(matches, PackageMatch{Package: pack, Repo: r}) } } @@ -569,3 +591,15 @@ func (re Repositories) Search(s string) []PackageMatch { return matches } + +func (re Repositories) SearchLabelMatch(s string) []PackageMatch { + return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexLabel}) +} + +func (re Repositories) SearchLabel(s string) []PackageMatch { + return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SLabel}) +} + +func (re Repositories) Search(s string) []PackageMatch { + return re.SearchPackages(s, LuetSearchOpts{Pattern: s, Mode: SRegexPkg}) +} diff --git a/pkg/package/database.go b/pkg/package/database.go index 82477a4f..fe1f59c4 100644 --- a/pkg/package/database.go +++ b/pkg/package/database.go @@ -45,6 +45,9 @@ type PackageSet interface { World() []Package FindPackageCandidate(p Package) (Package, error) + FindPackageLabel(labelKey string) ([]Package, error) + FindPackageLabelMatch(pattern string) ([]Package, error) + FindPackageMatch(pattern string) ([]Package, error) } type PackageFile struct { diff --git a/pkg/package/database_boltdb.go b/pkg/package/database_boltdb.go index 65d7afd2..50fbb528 100644 --- a/pkg/package/database_boltdb.go +++ b/pkg/package/database_boltdb.go @@ -18,6 +18,7 @@ package pkg import ( "encoding/base64" "os" + "regexp" "strconv" "sync" "time" @@ -384,3 +385,61 @@ func (db *BoltDatabase) FindPackageVersions(p Package) ([]Package, error) { } return versionsInWorld, nil } + +func (db *BoltDatabase) FindPackageLabel(labelKey string) ([]Package, error) { + var ans []Package + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + if pack.HasLabel(labelKey) { + ans = append(ans, pack) + } + } + return ans, nil +} + +func (db *BoltDatabase) FindPackageLabelMatch(pattern string) ([]Package, error) { + var ans []Package + + re := regexp.MustCompile(pattern) + if re == nil { + return nil, errors.New("Invalid regex " + pattern + "!") + } + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + if pack.MatchLabel(re) { + ans = append(ans, pack) + } + } + + return ans, nil +} + +func (db *BoltDatabase) FindPackageMatch(pattern string) ([]Package, error) { + var ans []Package + + re := regexp.MustCompile(pattern) + if re == nil { + return nil, errors.New("Invalid regex " + pattern + "!") + } + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + + if re.MatchString(pack.GetCategory() + pack.GetName()) { + ans = append(ans, pack) + } + } + + return ans, nil +} diff --git a/pkg/package/database_mem.go b/pkg/package/database_mem.go index 2e56d051..6a8056be 100644 --- a/pkg/package/database_mem.go +++ b/pkg/package/database_mem.go @@ -18,6 +18,7 @@ package pkg import ( "encoding/base64" "encoding/json" + "regexp" "sync" "github.com/pkg/errors" @@ -358,3 +359,62 @@ func (db *InMemoryDatabase) FindPackageCandidate(p Package) (Package, error) { return required, err } + +func (db *InMemoryDatabase) FindPackageLabel(labelKey string) ([]Package, error) { + var ans []Package + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + if pack.HasLabel(labelKey) { + ans = append(ans, pack) + } + } + + return ans, nil +} + +func (db *InMemoryDatabase) FindPackageLabelMatch(pattern string) ([]Package, error) { + var ans []Package + + re := regexp.MustCompile(pattern) + if re == nil { + return nil, errors.New("Invalid regex " + pattern + "!") + } + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + if pack.MatchLabel(re) { + ans = append(ans, pack) + } + } + + return ans, nil +} + +func (db *InMemoryDatabase) FindPackageMatch(pattern string) ([]Package, error) { + var ans []Package + + re := regexp.MustCompile(pattern) + if re == nil { + return nil, errors.New("Invalid regex " + pattern + "!") + } + + for _, k := range db.GetPackages() { + pack, err := db.GetPackage(k) + if err != nil { + return ans, err + } + + if re.MatchString(pack.GetCategory() + pack.GetName()) { + ans = append(ans, pack) + } + } + + return ans, nil +} diff --git a/pkg/package/package.go b/pkg/package/package.go index 4dc48e1c..22584e47 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -48,6 +48,7 @@ type Package interface { Requires([]*DefaultPackage) Package Conflicts([]*DefaultPackage) Package Revdeps(PackageDatabase) []Package + LabelDeps(PackageDatabase, string) []Package GetProvides() []*DefaultPackage SetProvides([]*DefaultPackage) Package @@ -85,6 +86,11 @@ type Package interface { SetLicense(string) GetLicense() string + AddLabel(string, string) + GetLabels() map[string]string + HasLabel(string) bool + MatchLabel(*regexp.Regexp) bool + IsSelector() bool VersionMatchSelector(string) (bool, error) SelectorMatchVersion(string) (bool, error) @@ -151,9 +157,11 @@ type DefaultPackage struct { // Path is set only internally when tree is loaded from disk Path string `json:"path,omitempty"` - Description string `json:"description"` - Uri []string `json:"uri"` - License string `json:"license"` + Description string `json:"description,omitempty"` + Uri []string `json:"uri,omitempty"` + License string `json:"license,omitempty"` + + Labels map[string]string `json:labels,omitempty` } // State represent the package state @@ -161,7 +169,13 @@ type State string // NewPackage returns a new package func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*DefaultPackage) *DefaultPackage { - return &DefaultPackage{Name: name, Version: version, PackageRequires: requires, PackageConflicts: conflicts} + return &DefaultPackage{ + Name: name, + Version: version, + PackageRequires: requires, + PackageConflicts: conflicts, + Labels: make(map[string]string, 0), + } } func (p *DefaultPackage) String() string { @@ -219,6 +233,28 @@ func (p *DefaultPackage) IsSelector() bool { return strings.ContainsAny(p.GetVersion(), "<>=") } +func (p *DefaultPackage) HasLabel(label string) bool { + ans := false + for k, _ := range p.Labels { + if k == label { + ans = true + break + } + } + return ans +} + +func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool { + ans := false + for k, v := range p.Labels { + if r.MatchString(k + "=" + v) { + ans = true + break + } + } + return ans +} + // AddUse adds a use to a package func (p *DefaultPackage) AddUse(use string) { for _, v := range p.UseFlags { @@ -303,6 +339,12 @@ func (p *DefaultPackage) SetCategory(s string) { func (p *DefaultPackage) GetUses() []string { return p.UseFlags } +func (p *DefaultPackage) AddLabel(k, v string) { + p.Labels[k] = v +} +func (p *DefaultPackage) GetLabels() map[string]string { + return p.Labels +} func (p *DefaultPackage) GetProvides() []*DefaultPackage { return p.Provides } @@ -373,6 +415,19 @@ func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) []Package { return versionsInWorld } +func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) []Package { + var pkgsWithLabelInWorld []Package + // TODO: check if integrate some index to improve + // research instead of iterate all list. + for _, w := range definitiondb.World() { + if w.HasLabel(labelKey) { + pkgsWithLabelInWorld = append(pkgsWithLabelInWorld, w) + } + } + + return pkgsWithLabelInWorld +} + func DecodePackage(ID string, db PackageDatabase) (Package, error) { return db.GetPackage(ID) } diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index d33c10b4..b007a34c 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -16,6 +16,8 @@ package pkg_test import ( + "regexp" + . "github.com/mudler/luet/pkg/package" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -61,6 +63,34 @@ var _ = Describe("Package", func() { }) }) + Context("Find label on packages", func() { + a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{}) + a.AddLabel("project1", "test1") + a.AddLabel("label2", "value1") + b := NewPackage("B", "1.0", []*DefaultPackage{}, []*DefaultPackage{}) + b.AddLabel("project2", "test2") + b.AddLabel("label2", "value1") + It("Expands correctly", func() { + var err error + definitions := NewInMemoryDatabase(false) + for _, p := range []Package{a, b} { + _, err = definitions.CreatePackage(p) + Expect(err).ToNot(HaveOccurred()) + } + re := regexp.MustCompile("project[0-9][=].*") + Expect(err).ToNot(HaveOccurred()) + Expect(re).ToNot(BeNil()) + Expect(a.HasLabel("label2")).To(Equal(true)) + Expect(a.HasLabel("label3")).To(Equal(false)) + Expect(a.HasLabel("project1")).To(Equal(true)) + Expect(b.HasLabel("project2")).To(Equal(true)) + Expect(b.HasLabel("label2")).To(Equal(true)) + Expect(b.MatchLabel(re)).To(Equal(true)) + Expect(a.MatchLabel(re)).To(Equal(true)) + + }) + }) + Context("Check description", func() { a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{}) a.SetDescription("Description A") diff --git a/pkg/tree/tree_test.go b/pkg/tree/tree_test.go index c54c232e..e76706c7 100644 --- a/pkg/tree/tree_test.go +++ b/pkg/tree/tree_test.go @@ -152,4 +152,21 @@ var _ = Describe("Tree", func() { }) }) + Context("Simple tree with labels", func() { + It("Read tree with labels", func() { + db := pkg.NewInMemoryDatabase(false) + generalRecipe := NewCompilerRecipe(db) + + err := generalRecipe.Load("../../tests/fixtures/labels") + Expect(err).ToNot(HaveOccurred()) + + Expect(len(generalRecipe.GetDatabase().World())).To(Equal(1)) + pack, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "pkgA", Category: "test", Version: "0.1"}) + Expect(err).ToNot(HaveOccurred()) + + Expect(pack.HasLabel("label1")).To(Equal(true)) + Expect(pack.HasLabel("label3")).To(Equal(false)) + }) + }) + }) diff --git a/tests/fixtures/labels/pkgA/0.1/build.yaml b/tests/fixtures/labels/pkgA/0.1/build.yaml new file mode 100644 index 00000000..6c7e610a --- /dev/null +++ b/tests/fixtures/labels/pkgA/0.1/build.yaml @@ -0,0 +1,3 @@ +image: "alpine" +steps: + - echo "test" > /file1 diff --git a/tests/fixtures/labels/pkgA/0.1/definition.yaml b/tests/fixtures/labels/pkgA/0.1/definition.yaml new file mode 100644 index 00000000..1bda91a2 --- /dev/null +++ b/tests/fixtures/labels/pkgA/0.1/definition.yaml @@ -0,0 +1,6 @@ +category: "test" +name: "pkgA" +version: "0.1" +labels: + label1: "value1" + label2: "value2"