// Copyright © 2019 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License along // with this program; if not, see . package pkg import ( "bytes" "crypto/md5" "encoding/json" "fmt" "io" "path/filepath" "regexp" "strconv" "strings" "github.com/mudler/luet/pkg/helpers" version "github.com/mudler/luet/pkg/versioner" gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo" "github.com/crillab/gophersat/bf" "github.com/ghodss/yaml" "github.com/jinzhu/copier" "github.com/pkg/errors" ) // Package is a package interface (TBD) // FIXME: Currently some of the methods are returning DefaultPackages due to JSON serialization of the package type Package interface { Encode(PackageDatabase) (string, error) Related(definitiondb PackageDatabase) Packages BuildFormula(PackageDatabase, PackageDatabase) ([]bf.Formula, error) GetFingerPrint() string GetPackageName() string GetPackageImageName() string Requires([]*DefaultPackage) Package Conflicts([]*DefaultPackage) Package Revdeps(PackageDatabase) Packages LabelDeps(PackageDatabase, string) Packages GetProvides() []*DefaultPackage SetProvides([]*DefaultPackage) Package GetRequires() []*DefaultPackage GetConflicts() []*DefaultPackage Expand(PackageDatabase) (Packages, error) SetCategory(string) GetName() string SetName(string) GetCategory() string GetVersion() string SetVersion(string) RequiresContains(PackageDatabase, Package) (bool, error) Matches(m Package) bool BumpBuildVersion() error AddUse(use string) RemoveUse(use string) GetUses() []string Yaml() ([]byte, error) Explain() SetPath(string) GetPath() string Rel(string) string GetDescription() string SetDescription(string) AddURI(string) GetURI() []string SetLicense(string) GetLicense() string AddLabel(string, string) GetLabels() map[string]string HasLabel(string) bool MatchLabel(*regexp.Regexp) bool AddAnnotation(string, string) GetAnnotations() map[string]string HasAnnotation(string) bool MatchAnnotation(*regexp.Regexp) bool IsHidden() bool IsSelector() bool VersionMatchSelector(string, version.Versioner) (bool, error) SelectorMatchVersion(string, version.Versioner) (bool, error) String() string HumanReadableString() string HashFingerprint(string) string SetBuildTimestamp(s string) GetBuildTimestamp() string Clone() Package } type Tree interface { GetPackageSet() PackageDatabase Prelude() string // A tree might have a prelude to be able to consume a tree SetPackageSet(s PackageDatabase) World() (Packages, error) FindPackage(Package) (Package, error) } type Packages []Package // >> Unmarshallers // DefaultPackageFromYaml decodes a package from yaml bytes func DefaultPackageFromYaml(yml []byte) (DefaultPackage, error) { var unescaped DefaultPackage source, err := yaml.YAMLToJSON(yml) if err != nil { return DefaultPackage{}, err } rawIn := json.RawMessage(source) bytes, err := rawIn.MarshalJSON() if err != nil { return DefaultPackage{}, err } err = json.Unmarshal(bytes, &unescaped) if err != nil { return DefaultPackage{}, err } return unescaped, nil } type rawPackages []map[string]interface{} func (r rawPackages) Find(name, category, version string) map[string]interface{} { for _, v := range r { if v["name"] == name && v["category"] == category && v["version"] == version { return v } } return map[string]interface{}{} } func GetRawPackages(yml []byte) (rawPackages, error) { var rawPackages struct { Packages []map[string]interface{} `yaml:"packages"` } source, err := yaml.YAMLToJSON(yml) if err != nil { return []map[string]interface{}{}, err } rawIn := json.RawMessage(source) bytes, err := rawIn.MarshalJSON() if err != nil { return []map[string]interface{}{}, err } err = json.Unmarshal(bytes, &rawPackages) if err != nil { return []map[string]interface{}{}, err } return rawPackages.Packages, nil } func DefaultPackagesFromYaml(yml []byte) ([]DefaultPackage, error) { var unescaped struct { Packages []DefaultPackage `json:"packages"` } source, err := yaml.YAMLToJSON(yml) if err != nil { return []DefaultPackage{}, err } rawIn := json.RawMessage(source) bytes, err := rawIn.MarshalJSON() if err != nil { return []DefaultPackage{}, err } err = json.Unmarshal(bytes, &unescaped) if err != nil { return []DefaultPackage{}, err } return unescaped.Packages, nil } // Major and minor gets escaped when marshalling in JSON, making compiler fails recognizing selectors for expansion func (t *DefaultPackage) JSON() ([]byte, error) { buffer := &bytes.Buffer{} encoder := json.NewEncoder(buffer) encoder.SetEscapeHTML(false) err := encoder.Encode(t) return buffer.Bytes(), err } // DefaultPackage represent a standard package definition type DefaultPackage struct { ID int `storm:"id,increment" json:"id"` // primary key with auto increment Name string `json:"name"` // Affects YAML field names too. Version string `json:"version"` // Affects YAML field names too. Category string `json:"category"` // Affects YAML field names too. UseFlags []string `json:"use_flags,omitempty"` // Affects YAML field names too. State State `json:"state,omitempty"` PackageRequires []*DefaultPackage `json:"requires"` // Affects YAML field names too. PackageConflicts []*DefaultPackage `json:"conflicts"` // Affects YAML field names too. Provides []*DefaultPackage `json:"provides,omitempty"` // Affects YAML field names too. Hidden bool `json:"hidden,omitempty"` // Affects YAML field names too. // Annotations are used for core features/options Annotations map[string]string `json:"annotations,omitempty"` // Affects YAML field names too // Path is set only internally when tree is loaded from disk Path string `json:"path,omitempty"` Description string `json:"description,omitempty"` Uri []string `json:"uri,omitempty"` License string `json:"license,omitempty"` BuildTimestamp string `json:"buildtimestamp,omitempty"` Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too. } // State represent the package state type State string // NewPackage returns a new package func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*DefaultPackage) *DefaultPackage { return &DefaultPackage{ Name: name, Version: version, PackageRequires: requires, PackageConflicts: conflicts, Labels: nil, } } func (p *DefaultPackage) String() string { b, err := p.JSON() if err != nil { return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\" }", p.ID, p.Name, p.Version, p.Category) } return fmt.Sprintf("%s", string(b)) } // GetFingerPrint returns a UUID of the package. // FIXME: this needs to be unique, now just name is generalized func (p *DefaultPackage) GetFingerPrint() string { return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version) } func (p *DefaultPackage) HashFingerprint(salt string) string { h := md5.New() io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt)) return fmt.Sprintf("%x", h.Sum(nil)) } func (p *DefaultPackage) HumanReadableString() string { return fmt.Sprintf("%s/%s-%s", p.Category, p.Name, p.Version) } func FromString(s string) Package { var unescaped DefaultPackage err := json.Unmarshal([]byte(s), &unescaped) if err != nil { return &unescaped } return &unescaped } func (p *DefaultPackage) GetPackageName() string { return fmt.Sprintf("%s-%s", p.Name, p.Category) } func (p *DefaultPackage) GetPackageImageName() string { return fmt.Sprintf("%s-%s:%s", p.Name, p.Category, p.Version) } // GetBuildTimestamp returns the package build timestamp func (p *DefaultPackage) GetBuildTimestamp() string { return p.BuildTimestamp } // SetBuildTimestamp sets the package Build timestamp func (p *DefaultPackage) SetBuildTimestamp(s string) { p.BuildTimestamp = s } // GetPath returns the path where the definition file was found func (p *DefaultPackage) GetPath() string { return p.Path } func (p *DefaultPackage) Rel(s string) string { return filepath.Join(p.GetPath(), s) } func (p *DefaultPackage) SetPath(s string) { p.Path = s } func (p *DefaultPackage) IsSelector() bool { return strings.ContainsAny(p.GetVersion(), "<>=") } func (p *DefaultPackage) IsHidden() bool { return p.Hidden } func (p *DefaultPackage) HasLabel(label string) bool { return helpers.MapHasKey(&p.Labels, label) } func (p *DefaultPackage) MatchLabel(r *regexp.Regexp) bool { return helpers.MapMatchRegex(&p.Labels, r) } func (p *DefaultPackage) HasAnnotation(label string) bool { return helpers.MapHasKey(&p.Annotations, label) } func (p *DefaultPackage) MatchAnnotation(r *regexp.Regexp) bool { return helpers.MapMatchRegex(&p.Annotations, r) } // AddUse adds a use to a package func (p *DefaultPackage) AddUse(use string) { for _, v := range p.UseFlags { if v == use { return } } p.UseFlags = append(p.UseFlags, use) } // RemoveUse removes a use to a package func (p *DefaultPackage) RemoveUse(use string) { for i := len(p.UseFlags) - 1; i >= 0; i-- { if p.UseFlags[i] == use { p.UseFlags = append(p.UseFlags[:i], p.UseFlags[i+1:]...) } } } // Encode encodes the package to string. // It returns an ID which can be used to retrieve the package later on. func (p *DefaultPackage) Encode(db PackageDatabase) (string, error) { return db.CreatePackage(p) } func (p *DefaultPackage) Yaml() ([]byte, error) { j, err := p.JSON() if err != nil { return []byte{}, err } y, err := yaml.JSONToYAML(j) if err != nil { return []byte{}, err } return y, nil } func (p *DefaultPackage) GetName() string { return p.Name } func (p *DefaultPackage) GetVersion() string { return p.Version } func (p *DefaultPackage) SetVersion(v string) { p.Version = v } func (p *DefaultPackage) GetDescription() string { return p.Description } func (p *DefaultPackage) SetDescription(s string) { p.Description = s } func (p *DefaultPackage) GetLicense() string { return p.License } func (p *DefaultPackage) SetLicense(s string) { p.License = s } func (p *DefaultPackage) AddURI(s string) { p.Uri = append(p.Uri, s) } func (p *DefaultPackage) GetURI() []string { return p.Uri } func (p *DefaultPackage) GetCategory() string { return p.Category } func (p *DefaultPackage) SetCategory(s string) { p.Category = s } func (p *DefaultPackage) SetName(s string) { p.Name = s } func (p *DefaultPackage) GetUses() []string { return p.UseFlags } func (p *DefaultPackage) AddLabel(k, v string) { if p.Labels == nil { p.Labels = make(map[string]string, 0) } p.Labels[k] = v } func (p *DefaultPackage) AddAnnotation(k, v string) { if p.Annotations == nil { p.Annotations = make(map[string]string, 0) } p.Annotations[k] = v } func (p *DefaultPackage) GetLabels() map[string]string { return p.Labels } func (p *DefaultPackage) GetAnnotations() map[string]string { return p.Annotations } func (p *DefaultPackage) GetProvides() []*DefaultPackage { return p.Provides } func (p *DefaultPackage) SetProvides(req []*DefaultPackage) Package { p.Provides = req return p } func (p *DefaultPackage) GetRequires() []*DefaultPackage { return p.PackageRequires } func (p *DefaultPackage) GetConflicts() []*DefaultPackage { return p.PackageConflicts } func (p *DefaultPackage) Requires(req []*DefaultPackage) Package { p.PackageRequires = req return p } func (p *DefaultPackage) Conflicts(req []*DefaultPackage) Package { p.PackageConflicts = req return p } func (p *DefaultPackage) Clone() Package { new := &DefaultPackage{} copier.Copy(&new, &p) return new } func (p *DefaultPackage) Matches(m Package) bool { if p.GetFingerPrint() == m.GetFingerPrint() { return true } return false } func (p *DefaultPackage) Expand(definitiondb PackageDatabase) (Packages, error) { var versionsInWorld Packages all, err := definitiondb.FindPackages(p) if err != nil { return nil, err } for _, w := range all { match, err := p.SelectorMatchVersion(w.GetVersion(), nil) if err != nil { return nil, err } if match { versionsInWorld = append(versionsInWorld, w) } } return versionsInWorld, nil } func (p *DefaultPackage) Revdeps(definitiondb PackageDatabase) Packages { var versionsInWorld Packages for _, w := range definitiondb.World() { if w.Matches(p) { continue } for _, re := range w.GetRequires() { if re.Matches(p) { versionsInWorld = append(versionsInWorld, w) versionsInWorld = append(versionsInWorld, w.Revdeps(definitiondb)...) } } } return versionsInWorld } func walkPackage(p Package, definitiondb PackageDatabase, visited map[string]interface{}) Packages { var versionsInWorld Packages if _, ok := visited[p.HumanReadableString()]; ok { return versionsInWorld } visited[p.HumanReadableString()] = true revdeps, _ := definitiondb.GetRevdeps(p) for _, r := range revdeps { versionsInWorld = append(versionsInWorld, r) } if !p.IsSelector() { versionsInWorld = append(versionsInWorld, p) } for _, re := range p.GetRequires() { versions, _ := re.Expand(definitiondb) for _, r := range versions { versionsInWorld = append(versionsInWorld, r) versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...) } } for _, re := range p.GetConflicts() { versions, _ := re.Expand(definitiondb) for _, r := range versions { versionsInWorld = append(versionsInWorld, r) versionsInWorld = append(versionsInWorld, walkPackage(r, definitiondb, visited)...) } } return versionsInWorld.Unique() } func (p *DefaultPackage) Related(definitiondb PackageDatabase) Packages { return walkPackage(p, definitiondb, map[string]interface{}{}) } func (p *DefaultPackage) LabelDeps(definitiondb PackageDatabase, labelKey string) Packages { var pkgsWithLabelInWorld Packages // TODO: check if integrate some index to improve // research instead of iterate all list. for _, w := range definitiondb.World() { if w.HasLabel(labelKey) { pkgsWithLabelInWorld = append(pkgsWithLabelInWorld, w) } } return pkgsWithLabelInWorld } func DecodePackage(ID string, db PackageDatabase) (Package, error) { return db.GetPackage(ID) } func (pack *DefaultPackage) scanRequires(definitiondb PackageDatabase, s Package, visited map[string]interface{}) (bool, error) { if _, ok := visited[pack.HumanReadableString()]; ok { return false, nil } visited[pack.HumanReadableString()] = true p, err := definitiondb.FindPackage(pack) if err != nil { p = pack //relax things //return false, errors.Wrap(err, "Package not found in definition db") } for _, re := range p.GetRequires() { if re.Matches(s) { return true, nil } packages, _ := re.Expand(definitiondb) for _, pa := range packages { if pa.Matches(s) { return true, nil } } if contains, err := re.scanRequires(definitiondb, s, visited); err == nil && contains { return true, nil } } return false, nil } // RequiresContains recursively scans into the database packages dependencies to find a match with the given package // It is used by the solver during uninstall. func (pack *DefaultPackage) RequiresContains(definitiondb PackageDatabase, s Package) (bool, error) { return pack.scanRequires(definitiondb, s, make(map[string]interface{})) } // Best returns the best version of the package (the most bigger) from a list // Accepts a versioner interface to change the ordering policy. If null is supplied // It defaults to version.WrappedVersioner which supports both semver and debian versioning func (set Packages) Best(v version.Versioner) Package { if v == nil { v = &version.WrappedVersioner{} } var versionsMap map[string]Package = make(map[string]Package) if len(set) == 0 { panic("Best needs a list with elements") } versionsRaw := []string{} for _, p := range set { versionsRaw = append(versionsRaw, p.GetVersion()) versionsMap[p.GetVersion()] = p } sorted := v.Sort(versionsRaw) return versionsMap[sorted[len(sorted)-1]] } func (set Packages) Unique() Packages { var result Packages uniq := make(map[string]Package) for _, p := range set { uniq[p.GetFingerPrint()] = p } for _, p := range uniq { result = append(result, p) } return result } func (pack *DefaultPackage) buildFormula(definitiondb PackageDatabase, db PackageDatabase, visited map[string]interface{}) ([]bf.Formula, error) { if _, ok := visited[pack.HumanReadableString()]; ok { return nil, nil } visited[pack.HumanReadableString()] = true p, err := definitiondb.FindPackage(pack) if err != nil { p = pack // Relax failures and trust the def } encodedA, err := p.Encode(db) if err != nil { return nil, err } A := bf.Var(encodedA) var formulas []bf.Formula // Do conflict with other packages versions (if A is selected, then conflict with other versions of A) packages, _ := definitiondb.FindPackageVersions(p) if len(packages) > 0 { for _, cp := range packages { encodedB, err := cp.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) if !p.Matches(cp) { formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(A), bf.Not(B)))) } } } for _, requiredDef := range p.GetRequires() { required, err := definitiondb.FindPackage(requiredDef) if err != nil || requiredDef.IsSelector() { if err == nil { requiredDef = required.(*DefaultPackage) } packages, err := definitiondb.FindPackages(requiredDef) if err != nil || len(packages) == 0 { required = requiredDef } else { var ALO, priorityConstraints, priorityALO []bf.Formula // Try to prio best match // Force the solver to consider first our candidate (if does exists). // Then builds ALO and AMO for the requires. c, candidateErr := definitiondb.FindPackageCandidate(requiredDef) var C bf.Formula if candidateErr == nil { // We have a desired candidate, try to look a solution with that included first for _, o := range packages { encodedB, err := o.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) if !o.Matches(c) { priorityConstraints = append(priorityConstraints, bf.Not(B)) priorityALO = append(priorityALO, B) } } encodedC, err := c.Encode(db) if err != nil { return nil, err } C = bf.Var(encodedC) // Or the Candidate is true, or all the others might be not true // This forces the CDCL sat implementation to look first at a solution with C=true formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.And(C, bf.Or(priorityConstraints...)), bf.And(bf.Not(C), bf.Or(priorityALO...))))) } // AMO - At most one for _, o := range packages { encodedB, err := o.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) ALO = append(ALO, B) for _, i := range packages { encodedI, err := i.Encode(db) if err != nil { return nil, err } I := bf.Var(encodedI) if !o.Matches(i) { formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(bf.Not(I), bf.Not(B)))) } } } formulas = append(formulas, bf.Or(bf.Not(A), bf.Or(ALO...))) // ALO - At least one continue } } encodedB, err := required.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) formulas = append(formulas, bf.Or(bf.Not(A), B)) r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage f, err := r.buildFormula(definitiondb, db, visited) if err != nil { return nil, err } formulas = append(formulas, f...) } for _, requiredDef := range p.GetConflicts() { required, err := definitiondb.FindPackage(requiredDef) if err != nil || requiredDef.IsSelector() { if err == nil { requiredDef = required.(*DefaultPackage) } packages, err := definitiondb.FindPackages(requiredDef) if err != nil || len(packages) == 0 { required = requiredDef } else { if len(packages) == 1 { required = packages[0] } else { for _, p := range packages { encodedB, err := p.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) formulas = append(formulas, bf.Or(bf.Not(A), bf.Not(B))) r := p.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage f, err := r.buildFormula(definitiondb, db, visited) if err != nil { return nil, err } formulas = append(formulas, f...) } continue } } } encodedB, err := required.Encode(db) if err != nil { return nil, err } B := bf.Var(encodedB) formulas = append(formulas, bf.Or(bf.Not(A), bf.Not(B))) r := required.(*DefaultPackage) // We know since the implementation is DefaultPackage, that can be only DefaultPackage f, err := r.buildFormula(definitiondb, db, visited) if err != nil { return nil, err } formulas = append(formulas, f...) } return formulas, nil } func (pack *DefaultPackage) BuildFormula(definitiondb PackageDatabase, db PackageDatabase) ([]bf.Formula, error) { return pack.buildFormula(definitiondb, db, make(map[string]interface{})) } func (p *DefaultPackage) Explain() { fmt.Println("====================") fmt.Println("Name: ", p.GetName()) fmt.Println("Category: ", p.GetCategory()) fmt.Println("Version: ", p.GetVersion()) for _, req := range p.GetRequires() { fmt.Println("\t-> ", req) } for _, con := range p.GetConflicts() { fmt.Println("\t!! ", con) } fmt.Println("====================") } func (p *DefaultPackage) BumpBuildVersion() error { cat := p.Category if cat == "" { // Use fake category for parse package cat = "app" } gp, err := gentoo.ParsePackageStr( fmt.Sprintf("%s/%s-%s", cat, p.Name, p.GetVersion())) if err != nil { return errors.Wrap(err, "Error on parser version") } buildPrefix := "" buildId := 0 if gp.VersionBuild != "" { // Check if version build is a number buildId, err = strconv.Atoi(gp.VersionBuild) if err == nil { goto end } // POST: is not only a number // TODO: check if there is a better way to handle all use cases. r1 := regexp.MustCompile(`^r[0-9]*$`) if r1 == nil { return errors.New("Error on create regex for -r[0-9]") } if r1.MatchString(gp.VersionBuild) { buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "r", "")) if err == nil { buildPrefix = "r" goto end } } p1 := regexp.MustCompile(`^p[0-9]*$`) if p1 == nil { return errors.New("Error on create regex for -p[0-9]") } if p1.MatchString(gp.VersionBuild) { buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "p", "")) if err == nil { buildPrefix = "p" goto end } } rc1 := regexp.MustCompile(`^rc[0-9]*$`) if rc1 == nil { return errors.New("Error on create regex for -rc[0-9]") } if rc1.MatchString(gp.VersionBuild) { buildId, err = strconv.Atoi(strings.ReplaceAll(gp.VersionBuild, "rc", "")) if err == nil { buildPrefix = "rc" goto end } } // Check if version build contains a dot dotIdx := strings.LastIndex(gp.VersionBuild, ".") if dotIdx > 0 { buildPrefix = gp.VersionBuild[0 : dotIdx+1] bVersion := gp.VersionBuild[dotIdx+1:] buildId, err = strconv.Atoi(bVersion) if err == nil { goto end } } buildPrefix = gp.VersionBuild + "." buildId = 0 } end: buildId++ p.Version = fmt.Sprintf("%s%s+%s%d", gp.Version, gp.VersionSuffix, buildPrefix, buildId) return nil } func (p *DefaultPackage) SelectorMatchVersion(ver string, v version.Versioner) (bool, error) { if !p.IsSelector() { return false, errors.New("Package is not a selector") } if v == nil { v = &version.WrappedVersioner{} } return v.ValidateSelector(ver, p.GetVersion()), nil } func (p *DefaultPackage) VersionMatchSelector(selector string, v version.Versioner) (bool, error) { if v == nil { v = &version.WrappedVersioner{} } return v.ValidateSelector(p.GetVersion(), selector), nil }