kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-07-13 15:14:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #56 from mudler/gorl

Browse Source 
Add resolvers to ehnance solver's heuristic
This commit is contained in:
geaaru 2020-02-12 15:39:56 +01:00 committed by GitHub
commit 0fb3497a3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 1376 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
README.md
View File

@ -30,6 +30,23 @@ To install luet, you can grab a release on the [Release page](https://github.com
Luet is not feature-complete yet, it can build, install/uninstall/upgrade packages - but it doesn't support yet all the features you would normally expect from a Package Manager nowadays.
# Dependency solving
Luet uses SAT and Reinforcement learning engine for dependency solving.
It encodes the package requirements into a SAT problem, using gophersat to solve the dependency tree and give a concrete model as result.
## SAT encoding
Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
## Reinforcement learning
Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
when they arises while trying to validate your queries against the system model.
To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
## Documentation
[Documentation](https://luet-lab.github.io/docs) is available, or

25
cmd/build.go
View File

@ -45,6 +45,11 @@ var buildCmd = &cobra.Command{
viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
viper.BindPFlag("all", cmd.Flags().Lookup("all"))
viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
},
Run: func(cmd *cobra.Command, args []string) {
@ -92,7 +97,22 @@ var buildCmd = &cobra.Command{
if err != nil {
Fatal("Error: " + err.Error())
}
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
LuetCfg.GetSolverOptions().Discount = float32(discount)
LuetCfg.GetSolverOptions().MaxAttempts = attempts
Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
opts := compiler.NewDefaultCompilerOptions()
opts.SolverOptions = *LuetCfg.GetSolverOptions()
opts.Clean = clean
luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts)
luetCompiler.SetConcurrency(concurrency)
@ -177,5 +197,10 @@ func init() {
buildCmd.Flags().String("destination", path, "Destination folder")
buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
buildCmd.Flags().String("solver-type", "", "Solver strategy")
buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
RootCmd.AddCommand(buildCmd)
}

6
cmd/cleanup.go
View File

@ -35,9 +35,9 @@ var cleanupCmd = &cobra.Command{
var cleaned int = 0
// Check if cache dir exists
if helpers.Exists(helpers.GetSystemPkgsCacheDirPath()) {
if helpers.Exists(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) {
files, err := ioutil.ReadDir(helpers.GetSystemPkgsCacheDirPath())
files, err := ioutil.ReadDir(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath())
if err != nil {
Fatal("Error on read cachedir ", err.Error())
}
@ -52,7 +52,7 @@ var cleanupCmd = &cobra.Command{
}
err := os.RemoveAll(
filepath.Join(helpers.GetSystemPkgsCacheDirPath(), file.Name()))
filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
if err != nil {
Fatal("Error on removing", file.Name())
}

25
cmd/install.go
View File

@ -22,7 +22,6 @@ import (
installer "github.com/mudler/luet/pkg/installer"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
@ -36,6 +35,10 @@ var installCmd = &cobra.Command{
PreRun: func(cmd *cobra.Command, args []string) {
LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
},
Long: `Install packages in parallel`,
Run: func(cmd *cobra.Command, args []string) {
@ -76,12 +79,24 @@ var installCmd = &cobra.Command{
repos = append(repos, r)
}
inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
LuetCfg.GetSolverOptions().Discount = float32(discount)
LuetCfg.GetSolverOptions().MaxAttempts = attempts
Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
inst.Repositories(repos)
if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
systemDB = pkg.NewBoltDatabase(
filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
} else {
systemDB = pkg.NewInMemoryDatabase(true)
}
@ -100,6 +115,10 @@ func init() {
}
installCmd.Flags().String("system-dbpath", path, "System db path")
installCmd.Flags().String("system-target", path, "System rootpath")
installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
RootCmd.AddCommand(installCmd)
}

3
cmd/repo/list.go
View File

@ -23,7 +23,6 @@ import (
"time"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
installer "github.com/mudler/luet/pkg/installer"
. "github.com/logrusorgru/aurora"
@ -69,7 +68,7 @@ func NewRepoListCommand() *cobra.Command {
repoText = Yellow(repo.Urls[0]).String()
}
repobasedir := helpers.GetRepoDatabaseDirPath(repo.Name)
repobasedir := LuetCfg.GetSystem().GetRepoDatabaseDirPath(repo.Name)
if repo.Cached {
r := installer.NewSystemRepository(repo)

30
cmd/search.go
View File

@ -20,13 +20,11 @@ import (
"regexp"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
installer "github.com/mudler/luet/pkg/installer"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var searchCmd = &cobra.Command{
@ -36,7 +34,11 @@ var searchCmd = &cobra.Command{
PreRun: func(cmd *cobra.Command, args []string) {
LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
LuetCfg.Viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
},
Run: func(cmd *cobra.Command, args []string) {
var systemDB pkg.PackageDatabase
@ -44,7 +46,18 @@ var searchCmd = &cobra.Command{
if len(args) != 1 {
Fatal("Wrong number of arguments (expected 1)")
}
installed := viper.GetBool("installed")
installed := LuetCfg.Viper.GetBool("installed")
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
LuetCfg.GetSolverOptions().Discount = float32(discount)
LuetCfg.GetSolverOptions().MaxAttempts = attempts
Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
if !installed {
@ -57,7 +70,8 @@ var searchCmd = &cobra.Command{
repos = append(repos, r)
}
inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
inst.Repositories(repos)
synced, err := inst.SyncRepositories(false)
if err != nil {
@ -75,7 +89,7 @@ var searchCmd = &cobra.Command{
if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
systemDB = pkg.NewBoltDatabase(
filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
} else {
systemDB = pkg.NewInMemoryDatabase(true)
}
@ -101,5 +115,9 @@ func init() {
searchCmd.Flags().String("system-dbpath", path, "System db path")
searchCmd.Flags().String("system-target", path, "System rootpath")
searchCmd.Flags().Bool("installed", false, "Search between system packages")
searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
RootCmd.AddCommand(searchCmd)
}

25
cmd/uninstall.go
View File

@ -20,7 +20,6 @@ import (
"path/filepath"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
installer "github.com/mudler/luet/pkg/installer"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
@ -36,6 +35,10 @@ var uninstallCmd = &cobra.Command{
PreRun: func(cmd *cobra.Command, args []string) {
LuetCfg.Viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
LuetCfg.Viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
},
Run: func(cmd *cobra.Command, args []string) {
var systemDB pkg.PackageDatabase
@ -60,11 +63,23 @@ var uninstallCmd = &cobra.Command{
Uri: make([]string, 0),
}
inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
LuetCfg.GetSolverOptions().Discount = float32(discount)
LuetCfg.GetSolverOptions().MaxAttempts = attempts
Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
systemDB = pkg.NewBoltDatabase(
filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
} else {
systemDB = pkg.NewInMemoryDatabase(true)
}
@ -84,5 +99,9 @@ func init() {
}
uninstallCmd.Flags().String("system-dbpath", path, "System db path")
uninstallCmd.Flags().String("system-target", path, "System rootpath")
uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
RootCmd.AddCommand(uninstallCmd)
}

26
cmd/upgrade.go
View File

@ -19,7 +19,6 @@ import (
"path/filepath"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
installer "github.com/mudler/luet/pkg/installer"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
@ -33,6 +32,10 @@ var upgradeCmd = &cobra.Command{
PreRun: func(cmd *cobra.Command, args []string) {
LuetCfg.Viper.BindPFlag("system.database_path", installCmd.Flags().Lookup("system-dbpath"))
LuetCfg.Viper.BindPFlag("system.rootfs", installCmd.Flags().Lookup("system-target"))
LuetCfg.Viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
LuetCfg.Viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
LuetCfg.Viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
LuetCfg.Viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
},
Long: `Upgrades packages in parallel`,
Run: func(cmd *cobra.Command, args []string) {
@ -48,7 +51,19 @@ var upgradeCmd = &cobra.Command{
repos = append(repos, r)
}
inst := installer.NewLuetInstaller(LuetCfg.GetGeneral().Concurrency)
stype := LuetCfg.Viper.GetString("solver.type")
discount := LuetCfg.Viper.GetFloat64("solver.discount")
rate := LuetCfg.Viper.GetFloat64("solver.rate")
attempts := LuetCfg.Viper.GetInt("solver.max_attempts")
LuetCfg.GetSolverOptions().Type = stype
LuetCfg.GetSolverOptions().LearnRate = float32(rate)
LuetCfg.GetSolverOptions().Discount = float32(discount)
LuetCfg.GetSolverOptions().MaxAttempts = attempts
Debug("Solver", LuetCfg.GetSolverOptions().String())
inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{Concurrency: LuetCfg.GetGeneral().Concurrency, SolverOptions: *LuetCfg.GetSolverOptions()})
inst.Repositories(repos)
_, err := inst.SyncRepositories(false)
if err != nil {
@ -57,7 +72,7 @@ var upgradeCmd = &cobra.Command{
if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {
systemDB = pkg.NewBoltDatabase(
filepath.Join(helpers.GetSystemRepoDatabaseDirPath(), "luet.db"))
filepath.Join(LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
} else {
systemDB = pkg.NewInMemoryDatabase(true)
}
@ -76,6 +91,9 @@ func init() {
}
upgradeCmd.Flags().String("system-dbpath", path, "System db path")
upgradeCmd.Flags().String("system-target", path, "System rootpath")
upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+AvailableResolvers+" )")
upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
RootCmd.AddCommand(upgradeCmd)
}

18
contrib/config/luet.yaml
View File

@ -108,3 +108,21 @@
# basic: "mybasicauth"
# Define token authentication header
# token: "mytoken"
# ---------------------------------------------
# Solver parameter configuration:
# ---------------------------------------------
# solver:
#
# Solver strategy to solve possible conflicts during depedency
# solving. Defaults to empty (none). Available: qlearning
# type: ""
#
# Solver agent learning rate. 0.1 to 1.0
# rate: 0.7
#
# Learning discount factor.
# discount: 1.0
#
# Number of overall attempts that the solver has available before bailing out.
# max_attempts: 9000
#

8
go.mod
View File

@ -4,16 +4,14 @@ go 1.12
require (
github.com/DataDog/zstd v1.4.4 // indirect
github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7
github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
github.com/briandowns/spinner v1.7.0
github.com/cavaliercoder/grab v2.0.0+incompatible
github.com/creack/pty v1.1.9 // indirect
github.com/crillab/gophersat v1.1.7
github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
github.com/ghodss/yaml v1.0.0
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
github.com/hashicorp/go-version v1.2.0
github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
github.com/klauspost/pgzip v1.2.1
@ -29,7 +27,6 @@ require (
github.com/pelletier/go-toml v1.6.0 // indirect
github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
github.com/pkg/errors v0.8.1
github.com/rogpeppe/go-internal v1.5.1 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
@ -46,6 +43,5 @@ require (
golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
gopkg.in/yaml.v2 v2.2.7
mvdan.cc/sh v2.6.4+incompatible // indirect
mvdan.cc/sh/v3 v3.0.0-beta1
)

28
go.sum
View File

@ -9,15 +9,9 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 h1:XCZM9J5KqLsr5NqtrZuXiD3X5fe5IfgU7IIUZzpeFBk=
github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56/go.mod h1:+Gbv6dg6TPHWq4oDjZY1vn978PLCEZ2hOu8kvn+S7t4=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Sabayon/pkgs-checker v0.4.1 h1:NImZhA5Z9souyr9Ff3nDzP0Bs9SGuDLxRzduVqci3dQ=
github.com/Sabayon/pkgs-checker v0.4.1/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
github.com/Sabayon/pkgs-checker v0.4.2-0.20191225230938-5ebdcd474e55 h1:rblyWFJ61sx69WwRsHK/BHcBlFm5X38sQmJnzVhyUoE=
github.com/Sabayon/pkgs-checker v0.4.2-0.20191225230938-5ebdcd474e55/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7 h1:Vf80sSLu1ZWjjMmUKhw0FqM43lEOvT8O5B22NaHB6AQ=
github.com/Sabayon/pkgs-checker v0.4.2-0.20200101193228-1d500105afb7/go.mod h1:GFGM6ZzSE5owdGgjLnulj0+Vt9UTd5LFGmB2AOVPYrE=
github.com/Sereal/Sereal v0.0.0-20181211220259-509a78ddbda3 h1:Xu7z47ZiE/J+sKXHZMGxEor/oY2q6dq51fkO0JqdSwY=
@ -59,9 +53,10 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crillab/gophersat v1.1.7 h1:f2Phe0W9jGyN1OefygKdcTdNM99q/goSjbWrFRjZGWc=
github.com/crillab/gophersat v1.1.7/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3 h1:HO63LCf9kTXQgUnlvFeS2qSDQhZ/cLP8DAJO89CythY=
github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3/go.mod h1:S91tHga1PCZzYhCkStwZAhvp1rCc+zqtSi55I+vDWGc=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -82,6 +77,8 @@ github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241 h1:+ebE/
github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd h1:JWEotl+g5uCCn37eVAYLF3UjBqO5HJ0ezZ5Zgnsdoqc=
github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd/go.mod h1:y0+kb0ORo7mC8lQbUzC4oa7ufu565J6SyUgWd39Z1Ic=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -94,8 +91,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
@ -152,6 +147,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
@ -204,7 +200,9 @@ github.com/opencontainers/runtime-spec v1.0.1 h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhz
github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
@ -231,7 +229,6 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.5.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4=
github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@ -266,8 +263,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b h1:wJSBFlabo96ySlmSX0a02WAPyGxagzTo9c5sk3sHP3E=
@ -326,8 +321,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 h1:ZC1Xn5A1nlpSmQCIva4bZ3ob3lmhYIefc+GU+DLg1Ow=
golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -336,7 +329,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -372,8 +364,6 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 h1:Hynbrlo6LbYI3H1IqXpkVDOcX/3HiPdhVEuyj5a59RM=
golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY=
golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -419,8 +409,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
@ -429,7 +417,5 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
mvdan.cc/editorconfig v0.1.1-0.20191109213504-890940e3f00e/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8=
mvdan.cc/sh v2.6.4+incompatible h1:eD6tDeh0pw+/TOTI1BBEryZ02rD2nMcFsgcvde7jffM=
mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
mvdan.cc/sh/v3 v3.0.0-beta1 h1:UqiwBEXEPzelaGxuvixaOtzc7WzKtrElePJ8HqvW7K8=
mvdan.cc/sh/v3 v3.0.0-beta1/go.mod h1:rBIndNJFYPp8xSppiZcGIk6B5d1g3OEARxEaXjPxwVI=

4
pkg/compiler/compiler.go
View File

@ -42,6 +42,7 @@ type LuetCompiler struct {
PullFirst, KeepImg, Clean bool
Concurrency int
CompressionType CompressionImplementation
Options CompilerOptions
}
func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler {
@ -58,6 +59,7 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *Compi
KeepImg: opt.KeepImg,
Concurrency: opt.Concurrency,
Clean: opt.Clean,
Options: *opt,
}
}
@ -459,7 +461,7 @@ func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPerm
func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
s := solver.NewSolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false))
s := solver.NewResolver(pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
solution, err := s.Install([]pkg.Package{p.GetPackage()})
if err != nil {

3
pkg/compiler/interface.go
View File

@ -18,6 +18,7 @@ package compiler
import (
"runtime"
"github.com/mudler/luet/pkg/config"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
)
@ -48,6 +49,8 @@ type CompilerOptions struct {
Concurrency int
CompressionType CompressionImplementation
Clean bool
SolverOptions config.LuetSolverOptions
}
func NewDefaultCompilerOptions() *CompilerOptions {

91
pkg/config/config.go
View File

@ -19,14 +19,20 @@ package config
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"runtime"
"strings"
"time"
solver "github.com/mudler/luet/pkg/solver"
v "github.com/spf13/viper"
)
var LuetCfg = NewLuetConfig(v.GetViper())
var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
type LuetLoggingConfig struct {
Path string `mapstructure:"path"`
@ -44,6 +50,31 @@ type LuetGeneralConfig struct {
FatalWarns bool `mapstructure:"fatal_warnings"`
}
type LuetSolverOptions struct {
Type string `mapstructure:"type"`
LearnRate float32 `mapstructure:"rate"`
Discount float32 `mapstructure:"discount"`
MaxAttempts int `mapstructure:"max_attempts"`
}
func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
switch opts.Type {
case solver.QLearningResolverType:
if opts.LearnRate != 0.0 {
return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
}
return solver.SimpleQLearningSolver()
}
return &solver.DummyPackageResolver{}
}
func (opts *LuetSolverOptions) CompactString() string {
return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
}
type LuetSystemConfig struct {
DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
DatabasePath string `yaml:"database_path" mapstructure:"database_path"`
@ -51,6 +82,44 @@ type LuetSystemConfig struct {
PkgsCachePath string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
}
func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
dbpath = filepath.Join(dbpath, "repos/"+name)
err := os.MkdirAll(dbpath, os.ModePerm)
if err != nil {
panic(err)
}
return dbpath
}
func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
dbpath := filepath.Join(sc.Rootfs,
sc.DatabasePath)
err := os.MkdirAll(dbpath, os.ModePerm)
if err != nil {
panic(err)
}
return dbpath
}
func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
var cachepath string
if sc.PkgsCachePath != "" {
cachepath = sc.PkgsCachePath
} else {
// Create dynamic cache for test suites
cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
}
if filepath.IsAbs(cachepath) {
ans = cachepath
} else {
ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
}
return
}
type LuetRepository struct {
Name string `json:"name" yaml:"name" mapstructure:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
@ -112,6 +181,7 @@ type LuetConfig struct {
Logging LuetLoggingConfig `mapstructure:"logging"`
General LuetGeneralConfig `mapstructure:"general"`
System LuetSystemConfig `mapstructure:"system"`
Solver LuetSolverOptions `mapstructure:"solver"`
RepositoriesConfDir []string `mapstructure:"repos_confdir"`
CacheRepositories []LuetRepository `mapstructure:"repetitors"`
@ -154,6 +224,11 @@ func GenDefault(viper *v.Viper) {
viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
viper.SetDefault("cache_repositories", []string{})
viper.SetDefault("system_repositories", []string{})
viper.SetDefault("solver.type", "")
viper.SetDefault("solver.rate", 0.7)
viper.SetDefault("solver.discount", 1.0)
viper.SetDefault("solver.max_attempts", 9000)
}
func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
@ -172,6 +247,10 @@ func (c *LuetConfig) GetSystem() *LuetSystemConfig {
return &c.System
}
func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
return &c.Solver
}
func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
var ans *LuetRepository = nil
@ -188,6 +267,18 @@ func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
return ans, nil
}
func (c *LuetSolverOptions) String() string {
ans := fmt.Sprintf(`
solver:
type: %s
rate: %f
discount: %f
max_attempts: %d`, c.Type, c.LearnRate, c.Discount,
c.MaxAttempts)
return ans
}
func (c *LuetGeneralConfig) String() string {
ans := fmt.Sprintf(`
general:

4
pkg/helpers/archive.go
View File

@ -21,8 +21,6 @@ import (
"os"
"path/filepath"
. "github.com/mudler/luet/pkg/config"
"github.com/docker/docker/pkg/archive"
)
@ -61,7 +59,7 @@ func Untar(src, dest string, sameOwner bool) error {
}
defer in.Close()
if LuetCfg.GetGeneral().SameOwner {
if sameOwner {
// PRE: i have root privileged.
opts := &archive.TarOptions{

24
pkg/helpers/math.go Normal file
View File

@ -0,0 +1,24 @@
// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package helpers
func Factorial(n uint64) (result uint64) {
if n > 0 {
result = n * Factorial(n-1)
return result
}
return 1
}

63
pkg/helpers/repository.go
View File

@ -1,63 +0,0 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
// Daniele Rondina <geaaru@sabayonlinux.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package helpers
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/mudler/luet/pkg/config"
)
func GetRepoDatabaseDirPath(name string) string {
dbpath := filepath.Join(config.LuetCfg.GetSystem().Rootfs, config.LuetCfg.GetSystem().DatabasePath)
dbpath = filepath.Join(dbpath, "repos/"+name)
err := os.MkdirAll(dbpath, os.ModePerm)
if err != nil {
panic(err)
}
return dbpath
}
func GetSystemRepoDatabaseDirPath() string {
dbpath := filepath.Join(config.LuetCfg.GetSystem().Rootfs,
config.LuetCfg.GetSystem().DatabasePath)
err := os.MkdirAll(dbpath, os.ModePerm)
if err != nil {
panic(err)
}
return dbpath
}
func GetSystemPkgsCacheDirPath() (ans string) {
var cachepath string
if config.LuetCfg.GetSystem().PkgsCachePath != "" {
cachepath = config.LuetCfg.GetSystem().PkgsCachePath
} else {
// Create dynamic cache for test suites
cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
}
if filepath.IsAbs(cachepath) {
ans = cachepath
} else {
ans = filepath.Join(GetSystemRepoDatabaseDirPath(), cachepath)
}
return
}

3
pkg/installer/client/http.go
View File

@ -27,6 +27,7 @@ import (
. "github.com/mudler/luet/pkg/logger"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
"github.com/cavaliercoder/grab"
@ -70,7 +71,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
var temp string
artifactName := path.Base(artifact.GetPath())
cacheFile := filepath.Join(helpers.GetSystemPkgsCacheDirPath(), artifactName)
cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
ok := false
// Check if file is already in cache

3
pkg/installer/client/local.go
View File

@ -21,6 +21,7 @@ import (
"path"
"path/filepath"
"github.com/mudler/luet/pkg/config"
. "github.com/mudler/luet/pkg/logger"
"github.com/mudler/luet/pkg/compiler"
@ -39,7 +40,7 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art
var err error
artifactName := path.Base(artifact.GetPath())
cacheFile := filepath.Join(helpers.GetSystemPkgsCacheDirPath(), artifactName)
cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
// Check if file is already in cache
if helpers.Exists(cacheFile) {

23
pkg/installer/installer.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ghodss/yaml"
compiler "github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
@ -34,9 +35,15 @@ import (
"github.com/pkg/errors"
)
type LuetInstallerOptions struct {
SolverOptions config.LuetSolverOptions
Concurrency int
}
type LuetInstaller struct {
PackageRepositories Repositories
Concurrency int
Options LuetInstallerOptions
}
type ArtifactMatch struct {
@ -86,8 +93,8 @@ func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
return &p, err
}
func NewLuetInstaller(concurrency int) Installer {
return &LuetInstaller{Concurrency: concurrency}
func NewLuetInstaller(opts LuetInstallerOptions) Installer {
return &LuetInstaller{Options: opts}
}
func (l *LuetInstaller) Upgrade(s *System) error {
@ -100,7 +107,7 @@ func (l *LuetInstaller) Upgrade(s *System) error {
allRepos := pkg.NewInMemoryDatabase(false)
syncedRepos.SyncDatabase(allRepos)
// compute a "big" world
solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
uninstall, solution, err := solv.Upgrade()
if err != nil {
return errors.Wrap(err, "Failed solving solution for upgrade")
@ -179,7 +186,8 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
allRepos := pkg.NewInMemoryDatabase(false)
syncedRepos.SyncDatabase(allRepos)
p = syncedRepos.ResolveSelectors(p)
solv := solver.NewSolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false))
solv := solver.NewResolver(s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
solution, err := solv.Install(p)
if err != nil {
return errors.Wrap(err, "Failed solving solution for package")
@ -214,7 +222,7 @@ func (l *LuetInstaller) Install(cp []pkg.Package, s *System) error {
all := make(chan ArtifactMatch)
var wg = new(sync.WaitGroup)
for i := 0; i < l.Concurrency; i++ {
for i := 0; i < l.Options.Concurrency; i++ {
wg.Add(1)
go l.installerWorker(i, wg, all, s)
}
@ -356,8 +364,7 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
// compute uninstall from all world - remove packages in parallel - run uninstall finalizer (in order) - mark the uninstallation in db
// Get installed definition
solv := solver.NewSolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false))
solv := solver.NewResolver(s.Database, s.Database, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
solution, err := solv.Uninstall(p)
if err != nil {
return errors.Wrap(err, "Uninstall failed")

8
pkg/installer/installer_test.go
View File

@ -99,7 +99,7 @@ var _ = Describe("Installer", func() {
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(fakeroot) // clean up
inst := NewLuetInstaller(1)
inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
name: "test"
type: "disk"
@ -212,7 +212,7 @@ urls:
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(fakeroot) // clean up
inst := NewLuetInstaller(1)
inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
name: "test"
type: "disk"
@ -320,7 +320,7 @@ urls:
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(fakeroot) // clean up
inst := NewLuetInstaller(1)
inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
name: "test"
type: "disk"
@ -435,7 +435,7 @@ urls:
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(fakeroot) // clean up
inst := NewLuetInstaller(1)
inst := NewLuetInstaller(LuetInstallerOptions{Concurrency: 1})
repo2, err := NewLuetSystemRepositoryFromYaml([]byte(`
name: "test"
type: "disk"

2
pkg/installer/repository.go
View File

@ -360,7 +360,7 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE)
}
repobasedir := helpers.GetRepoDatabaseDirPath(r.GetName())
repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName())
repo, err := r.ReadSpecFile(file, false)
if err != nil {
return nil, err

16
pkg/package/package.go
View File

@ -85,6 +85,8 @@ type Package interface {
IsSelector() bool
VersionMatchSelector(string) (bool, error)
SelectorMatchVersion(string) (bool, error)
String() string
}
type Tree interface {
@ -128,7 +130,7 @@ func (t *DefaultPackage) JSON() ([]byte, error) {
// DefaultPackage represent a standard package definition
type DefaultPackage struct {
ID int `storm:"id,increment"` // primary key with auto increment
ID int `storm:"id,increment" json:"id"` // primary key with auto increment
Name string `json:"name"` // Affects YAML field names too.
Version string `json:"version"` // Affects YAML field names too.
Category string `json:"category"` // Affects YAML field names too.
@ -160,7 +162,7 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D
func (p *DefaultPackage) String() string {
b, err := p.JSON()
if err != nil {
return fmt.Sprintf("{ id: \"%d\", name: \"%s\" }", p.ID, p.Name)
return fmt.Sprintf("{ id: \"%d\", name: \"%s\", version: \"%s\", category: \"%s\" }", p.ID, p.Name, p.Version, p.Category)
}
return fmt.Sprintf("%s", string(b))
}
@ -171,6 +173,16 @@ func (p *DefaultPackage) GetFingerPrint() string {
return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version)
}
func FromString(s string) Package {
var unescaped DefaultPackage
err := json.Unmarshal([]byte(s), &unescaped)
if err != nil {
return &unescaped
}
return &unescaped
}
func (p *DefaultPackage) GetPackageName() string {
return fmt.Sprintf("%s-%s", p.Name, p.Category)
}

9
pkg/package/package_test.go
View File

@ -23,6 +23,15 @@ import (
var _ = Describe("Package", func() {
Context("Encoding/Decoding", func() {
a := &DefaultPackage{Name: "test", Version: "1", Category: "t"}
It("Encodes and decodes correctly", func() {
Expect(a.String()).ToNot(Equal(""))
p := FromString(a.String())
Expect(p).To(Equal(a))
})
})
Context("Simple package", func() {
a := NewPackage("A", ">=1.0", []*DefaultPackage{}, []*DefaultPackage{})
a1 := NewPackage("A", "1.0", []*DefaultPackage{}, []*DefaultPackage{})

336
pkg/solver/resolver.go Normal file
View File

@ -0,0 +1,336 @@
// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package solver
import (
"encoding/json"
"fmt"
"strconv"
"github.com/crillab/gophersat/bf"
"github.com/mudler/luet/pkg/helpers"
"gopkg.in/yaml.v2"
"github.com/ecooper/qlearning"
pkg "github.com/mudler/luet/pkg/package"
"github.com/pkg/errors"
)
type ActionType int
const (
NoAction = 0
Solved = iota
NoSolution = iota
Going = iota
ActionRemoved = iota
ActionAdded = iota
DoNoop = false
ActionDomains = 3 // Bump it if you increase the number of actions
DefaultMaxAttempts = 9000
DefaultLearningRate = 0.7
DefaultDiscount = 1.0
DefaultInitialObserved = 999999
QLearningResolverType = "qlearning"
)
//. "github.com/mudler/luet/pkg/logger"
// PackageResolver assists PackageSolver on unsat cases
type PackageResolver interface {
Solve(bf.Formula, PackageSolver) (PackagesAssertions, error)
}
type DummyPackageResolver struct {
}
func (*DummyPackageResolver) Solve(bf.Formula, PackageSolver) (PackagesAssertions, error) {
return nil, errors.New("Could not satisfy the constraints. Try again by removing deps ")
}
type QLearningResolver struct {
Attempts int
ToAttempt int
attempts int
Attempted map[string]bool
Solver PackageSolver
Formula bf.Formula
Targets []pkg.Package
Current []pkg.Package
observedDelta int
observedDeltaChoice []pkg.Package
Agent *qlearning.SimpleAgent
}
func SimpleQLearningSolver() PackageResolver {
return NewQLearningResolver(DefaultLearningRate, DefaultDiscount, DefaultMaxAttempts, DefaultInitialObserved)
}
// Defaults LearningRate 0.7, Discount 1.0
func NewQLearningResolver(LearningRate, Discount float32, MaxAttempts, initialObservedDelta int) PackageResolver {
return &QLearningResolver{
Agent: qlearning.NewSimpleAgent(LearningRate, Discount),
observedDelta: initialObservedDelta,
Attempts: MaxAttempts,
}
}
func (resolver *QLearningResolver) Solve(f bf.Formula, s PackageSolver) (PackagesAssertions, error) {
// Info("Using QLearning solver to resolve conflicts. Please be patient.")
resolver.Solver = s
s.SetResolver(&DummyPackageResolver{}) // Set dummy. Otherwise the attempts will run again a QLearning instance.
defer s.SetResolver(resolver) // Set back ourselves as resolver
resolver.Formula = f
// Our agent by default has a learning rate of 0.7 and discount of 1.0.
if resolver.Agent == nil {
resolver.Agent = qlearning.NewSimpleAgent(DefaultLearningRate, DefaultDiscount) // FIXME: Remove hardcoded values
}
// 3 are the action domains, counting noop regardless if enabled or not
// get the permutations to attempt
resolver.ToAttempt = int(helpers.Factorial(uint64(len(resolver.Solver.(*Solver).Wanted)-1) * ActionDomains)) // TODO: type assertions must go away
resolver.Targets = resolver.Solver.(*Solver).Wanted
resolver.attempts = resolver.Attempts
resolver.Attempted = make(map[string]bool, len(resolver.Targets))
for resolver.IsComplete() == Going {
// Pick the next move, which is going to be a letter choice.
action := qlearning.Next(resolver.Agent, resolver)
// Whatever that choice is, let's update our model for its
// impact. If the package chosen makes the formula sat,
// then this action will be positive. Otherwise, it will be
// negative.
resolver.Agent.Learn(action, resolver)
// Reward doesn't change state so we can check what the
// reward would be for this action, and report how the
// env changed.
// score := resolver.Reward(action)
// if score > 0.0 {
// resolver.Log("%s was correct", action.Action.String())
// } else {
// resolver.Log("%s was incorrect", action.Action.String())
// }
}
// If we get good result, take it
// Take the result also if we did reached overall maximum attempts
if resolver.IsComplete() == Solved || resolver.IsComplete() == NoSolution {
if len(resolver.observedDeltaChoice) != 0 {
// Take the minimum delta observed choice result, and consume it (Try sets the wanted list)
resolver.Solver.(*Solver).Wanted = resolver.observedDeltaChoice
}
return resolver.Solver.Solve()
} else {
return nil, errors.New("QLearning resolver failed ")
}
}
// Returns the current state.
func (resolver *QLearningResolver) IsComplete() int {
if resolver.attempts < 1 {
return NoSolution
}
if resolver.ToAttempt > 0 {
return Going
}
return Solved
}
func (resolver *QLearningResolver) Try(c Choice) error {
pack := c.Package
packtoAdd := pkg.FromString(pack)
resolver.Attempted[pack+strconv.Itoa(int(c.Action))] = true // increase the count
s, _ := resolver.Solver.(*Solver)
var filtered []pkg.Package
switch c.Action {
case ActionAdded:
found := false
for _, p := range s.Wanted {
if p.String() == pack {
found = true
break
}
}
if !found {
resolver.Solver.(*Solver).Wanted = append(resolver.Solver.(*Solver).Wanted, packtoAdd)
}
case ActionRemoved:
for _, p := range s.Wanted {
if p.String() != pack {
filtered = append(filtered, p)
}
}
resolver.Solver.(*Solver).Wanted = filtered
}
_, err := resolver.Solver.Solve()
return err
}
// Choose applies a pack attempt, returning
// true if the formula returns sat.
//
// Choose updates the resolver's state.
func (resolver *QLearningResolver) Choose(c Choice) bool {
//pack := pkg.FromString(c.Package)
err := resolver.Try(c)
if err == nil {
resolver.ToAttempt--
resolver.attempts-- // Decrease attempts - it's a barrier. We could also do not decrease it here, allowing more attempts to be made
} else {
resolver.attempts--
return false
}
return true
}
// Reward returns a score for a given qlearning.StateAction. Reward is a
// member of the qlearning.Rewarder interface. If the choice will make sat the formula, a positive score is returned.
// Otherwise, a static -1000 is returned.
func (resolver *QLearningResolver) Reward(action *qlearning.StateAction) float32 {
choice := action.Action.(*Choice)
//_, err := resolver.Solver.Solve()
err := resolver.Try(*choice)
toBeInstalled := len(resolver.Solver.(*Solver).Wanted)
originalTarget := len(resolver.Targets)
noaction := choice.Action == NoAction
delta := originalTarget - toBeInstalled
if err == nil {
// if toBeInstalled == originalTarget { // Base case: all the targets matches (it shouldn't happen, but lets put a higher)
// Debug("Target match, maximum score")
// return 24.0 / float32(len(resolver.Attempted))
// }
if DoNoop {
if noaction && toBeInstalled == 0 { // We decided to stay in the current state, and no targets have been chosen
return -100
}
}
if delta <= resolver.observedDelta { // Try to maximise observedDelta
resolver.observedDelta = delta
resolver.observedDeltaChoice = resolver.Solver.(*Solver).Wanted // we store it as this is our return value at the end
return 24.0 / float32(len(resolver.Attempted))
} else if toBeInstalled > 0 { // If we installed something, at least give a good score
return 24.0 / float32(len(resolver.Attempted))
}
}
return -1000
}
// Next creates a new slice of qlearning.Action instances. A possible
// action is created for each package that could be removed from the formula's target
func (resolver *QLearningResolver) Next() []qlearning.Action {
actions := make([]qlearning.Action, 0, (len(resolver.Targets)-1)*3)
TARGETS:
for _, pack := range resolver.Targets {
for _, current := range resolver.Solver.(*Solver).Wanted {
if current.String() == pack.String() {
actions = append(actions, &Choice{Package: pack.String(), Action: ActionRemoved})
continue TARGETS
}
}
actions = append(actions, &Choice{Package: pack.String(), Action: ActionAdded})
}
if DoNoop {
actions = append(actions, &Choice{Package: "", Action: NoAction}) // NOOP
}
return actions
}
// Log is a wrapper of fmt.Printf. If Game.debug is true, Log will print
// to stdout.
func (resolver *QLearningResolver) Log(msg string, args ...interface{}) {
logMsg := fmt.Sprintf("(%d moves, %d remaining attempts) %s\n", len(resolver.Attempted), resolver.attempts, msg)
fmt.Println(fmt.Sprintf(logMsg, args...))
}
// String returns a consistent hash for the current env state to be
// used in a qlearning.Agent.
func (resolver *QLearningResolver) String() string {
return fmt.Sprintf("%v", resolver.Solver.(*Solver).Wanted)
}
// Choice implements qlearning.Action for a package choice for removal from wanted targets
type Choice struct {
Package string `json:"pack"`
Action ActionType `json:"action"`
}
func ChoiceFromString(s string) (*Choice, error) {
var p *Choice
err := yaml.Unmarshal([]byte(s), &p)
if err != nil {
return nil, err
}
return p, nil
}
// String returns the character for the current action.
func (choice *Choice) String() string {
data, err := json.Marshal(choice)
if err != nil {
return ""
}
return string(data)
}
// Apply updates the state of the solver for the package choice.
func (choice *Choice) Apply(state qlearning.State) qlearning.State {
resolver := state.(*QLearningResolver)
resolver.Choose(*choice)
return resolver
}

182
pkg/solver/resolver_test.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package solver_test
import (
pkg "github.com/mudler/luet/pkg/package"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
. "github.com/mudler/luet/pkg/solver"
)
var _ = Describe("Resolver", func() {
db := pkg.NewInMemoryDatabase(false)
dbInstalled := pkg.NewInMemoryDatabase(false)
dbDefinitions := pkg.NewInMemoryDatabase(false)
s := NewSolver(dbInstalled, dbDefinitions, db)
BeforeEach(func() {
db = pkg.NewInMemoryDatabase(false)
dbInstalled = pkg.NewInMemoryDatabase(false)
dbDefinitions = pkg.NewInMemoryDatabase(false)
s = NewSolver(dbInstalled, dbDefinitions, db)
})
Context("Conflict set", func() {
Context("DummyPackageResolver", func() {
It("is unsolvable - as we something we ask to install conflict with system stuff", func() {
C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
for _, p := range []pkg.Package{A, B, C} {
_, err := dbDefinitions.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
for _, p := range []pkg.Package{C} {
_, err := dbInstalled.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
solution, err := s.Install([]pkg.Package{A})
Expect(len(solution)).To(Equal(0))
Expect(err).To(HaveOccurred())
})
It("succeeds to install D and F if explictly requested", func() {
C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
for _, p := range []pkg.Package{A, B, C, D, E, F} {
_, err := dbDefinitions.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
for _, p := range []pkg.Package{C} {
_, err := dbInstalled.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
solution, err := s.Install([]pkg.Package{D, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
Expect(err).ToNot(HaveOccurred())
Expect(len(solution)).To(Equal(6))
Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
})
})
Context("QLearningResolver", func() {
It("will find out that we can install D by ignoring A", func() {
s.SetResolver(SimpleQLearningSolver())
C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
for _, p := range []pkg.Package{A, B, C, D} {
_, err := dbDefinitions.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
for _, p := range []pkg.Package{C} {
_, err := dbInstalled.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
solution, err := s.Install([]pkg.Package{A, D})
Expect(err).ToNot(HaveOccurred())
Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true}))
Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
Expect(len(solution)).To(Equal(4))
})
It("will find out that we can install D and F by ignoring E and A", func() {
s.SetResolver(SimpleQLearningSolver())
C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
E := pkg.NewPackage("E", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
F := pkg.NewPackage("F", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
for _, p := range []pkg.Package{A, B, C, D, E, F} {
_, err := dbDefinitions.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
for _, p := range []pkg.Package{C} {
_, err := dbInstalled.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
solution, err := s.Install([]pkg.Package{A, D, E, F}) // D and F should go as they have no deps. A/E should be filtered by QLearn
Expect(err).ToNot(HaveOccurred())
Expect(solution).To(ContainElement(PackageAssert{Package: A, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: true})) // Was already installed
Expect(solution).To(ContainElement(PackageAssert{Package: D, Value: true}))
Expect(solution).To(ContainElement(PackageAssert{Package: E, Value: false}))
Expect(solution).To(ContainElement(PackageAssert{Package: F, Value: true}))
Expect(len(solution)).To(Equal(6))
})
})
Context("DummyPackageResolver", func() {
It("cannot find a solution", func() {
C := pkg.NewPackage("C", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
B := pkg.NewPackage("B", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{C})
A := pkg.NewPackage("A", "", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
D := pkg.NewPackage("D", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
for _, p := range []pkg.Package{A, B, C, D} {
_, err := dbDefinitions.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
for _, p := range []pkg.Package{C} {
_, err := dbInstalled.CreatePackage(p)
Expect(err).ToNot(HaveOccurred())
}
solution, err := s.Install([]pkg.Package{A, D})
Expect(err).To(HaveOccurred())
Expect(len(solution)).To(Equal(0))
})
})
})
})

31
pkg/solver/solver.go
View File

@ -33,6 +33,10 @@ type PackageSolver interface {
ConflictsWith(p pkg.Package, ls []pkg.Package) (bool, error)
World() []pkg.Package
Upgrade() ([]pkg.Package, PackagesAssertions, error)
SetResolver(PackageResolver)
Solve() (PackagesAssertions, error)
}
// Solver is the default solver for luet
@ -41,20 +45,33 @@ type Solver struct {
SolverDatabase pkg.PackageDatabase
Wanted []pkg.Package
InstalledDatabase pkg.PackageDatabase
Resolver PackageResolver
}
// NewSolver accepts as argument two lists of packages, the first is the initial set,
// the second represent all the known packages.
func NewSolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase) PackageSolver {
return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb}
return NewResolver(installed, definitiondb, solverdb, &DummyPackageResolver{})
}
// SetWorld is a setter for the list of all known packages to the solver
// NewReSolver accepts as argument two lists of packages, the first is the initial set,
// the second represent all the known packages.
func NewResolver(installed pkg.PackageDatabase, definitiondb pkg.PackageDatabase, solverdb pkg.PackageDatabase, re PackageResolver) PackageSolver {
return &Solver{InstalledDatabase: installed, DefinitionDatabase: definitiondb, SolverDatabase: solverdb, Resolver: re}
}
// SetDefinitionDatabase is a setter for the definition Database
func (s *Solver) SetDefinitionDatabase(db pkg.PackageDatabase) {
s.DefinitionDatabase = db
}
// SetResolver is a setter for the unsat resolver backend
func (s *Solver) SetResolver(r PackageResolver) {
s.Resolver = r
}
func (s *Solver) World() []pkg.Package {
return s.DefinitionDatabase.World()
}
@ -221,6 +238,7 @@ func (s *Solver) Upgrade() ([]pkg.Package, PackagesAssertions, error) {
}
s2 := NewSolver(installedcopy, s.DefinitionDatabase, pkg.NewInMemoryDatabase(false))
s2.SetResolver(s.Resolver)
// Then try to uninstall the versions in the system, and store that tree
for _, p := range toUninstall {
r, err := s.Uninstall(p)
@ -361,13 +379,20 @@ func (s *Solver) solve(f bf.Formula) (map[string]bool, bf.Formula, error) {
// Solve builds the formula given the current state and returns package assertions
func (s *Solver) Solve() (PackagesAssertions, error) {
var model map[string]bool
var err error
f, err := s.BuildFormula()
if err != nil {
return nil, err
}
model, _, err := s.solve(f)
model, _, err = s.solve(f)
if err != nil && s.Resolver != nil {
return s.Resolver.Solve(f, s)
}
if err != nil {
return nil, err
}

8
tests/fixtures/qlearning/a/build.yaml vendored Normal file
View File

@ -0,0 +1,8 @@
prelude:
- echo a > /aprelude
steps:
- echo a > /a
requires:
- category: "test"
name: "b"
version: "1.0"

8
tests/fixtures/qlearning/a/definition.yaml vendored Normal file
View File

@ -0,0 +1,8 @@
category: "test"
name: "a"
version: "1.0"
requires:
- category: "test"
name: "b"
version: "1.0"

5
tests/fixtures/qlearning/b/build.yaml vendored Normal file
View File

@ -0,0 +1,5 @@
image: "alpine"
prelude:
- echo b > /bprelude
steps:
- echo b > /b

7
tests/fixtures/qlearning/b/definition.yaml vendored Normal file
View File

@ -0,0 +1,7 @@
category: "test"
name: "b"
version: "1.0"
conflicts:
- category: "test"
name: "c"
version: "1.0"

5
tests/fixtures/qlearning/c/build.yaml vendored Normal file
View File

@ -0,0 +1,5 @@
image: "alpine"
prelude:
- echo c > /cprelude
steps:
- echo c > /c

3
tests/fixtures/qlearning/c/definition.yaml vendored Normal file
View File

@ -0,0 +1,3 @@
category: "test"
name: "c"
version: "1.0"

5
tests/fixtures/qlearning/d/build.yaml vendored Normal file
View File

@ -0,0 +1,5 @@
image: "alpine"
prelude:
- echo d > /dprelude
steps:
- echo d > /d

3
tests/fixtures/qlearning/d/definition.yaml vendored Normal file
View File

@ -0,0 +1,3 @@
category: "test"
name: "d"
version: "1.0"

9
tests/fixtures/qlearning/e/build.yaml vendored Normal file
View File

@ -0,0 +1,9 @@
prelude:
- echo e > /eprelude
steps:
- echo e > /e
requires:
- category: "test"
name: "b"
version: "1.0"

8
tests/fixtures/qlearning/e/definition.yaml vendored Normal file
View File

@ -0,0 +1,8 @@
category: "test"
name: "e"
version: "1.0"
requires:
- category: "test"
name: "b"
version: "1.0"

5
tests/fixtures/qlearning/f/build.yaml vendored Normal file
View File

@ -0,0 +1,5 @@
prelude:
- echo f > /eprelude
steps:
- echo f > /f
image: "alpine"

3
tests/fixtures/qlearning/f/definition.yaml vendored Normal file
View File

@ -0,0 +1,3 @@
category: "test"
name: "f"
version: "1.0"

94
tests/integration/03_qlearning.sh Executable file
View File

@ -0,0 +1,94 @@
#!/bin/bash
export LUET_NOLOCK=true
oneTimeSetUp() {
export tmpdir="$(mktemp -d)"
}
oneTimeTearDown() {
rm -rf "$tmpdir"
}
testBuild() {
mkdir $tmpdir/testbuild
luet build --all --tree "$ROOT_DIR/tests/fixtures/qlearning" --destination $tmpdir/testbuild --compression gzip
buildst=$?
assertEquals 'builds successfully' "$buildst" "0"
assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
}
testRepo() {
assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
luet create-repo --tree "$ROOT_DIR/tests/fixtures/qlearning" \
--output $tmpdir/testbuild \
--packages $tmpdir/testbuild \
--name "test" \
--descr "Test Repo" \
--urls $tmpdir/testrootfs \
--type disk > /dev/null
createst=$?
assertEquals 'create repo successfully' "$createst" "0"
assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
}
testConfig() {
mkdir $tmpdir/testrootfs
cat <<EOF > $tmpdir/luet.yaml
general:
debug: true
system:
rootfs: $tmpdir/testrootfs
database_path: "/"
database_engine: "boltdb"
repositories:
- name: "main"
type: "disk"
enable: true
urls:
- "$tmpdir/testbuild"
EOF
luet config --config $tmpdir/luet.yaml
res=$?
assertEquals 'config test successfully' "$res" "0"
}
testInstall() {
luet install --config $tmpdir/luet.yaml test/c
#luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package C installed' "[ -e '$tmpdir/testrootfs/c' ]"
}
testFullInstall() {
output=$(luet install --config $tmpdir/luet.yaml test/d test/f test/e test/a)
installst=$?
assertEquals 'cannot install' "$installst" "1"
assertTrue 'package D installed' "[ ! -e '$tmpdir/testrootfs/d' ]"
assertTrue 'package F installed' "[ ! -e '$tmpdir/testrootfs/f' ]"
}
testInstallAgain() {
output=$(luet install --solver-type qlearning --config $tmpdir/luet.yaml test/d test/f test/e test/a)
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertNotContains 'contains warning' "$output" 'Filtering out'
assertTrue 'package D installed' "[ -e '$tmpdir/testrootfs/d' ]"
assertTrue 'package F installed' "[ -e '$tmpdir/testrootfs/f' ]"
assertTrue 'package E not installed' "[ ! -e '$tmpdir/testrootfs/e' ]"
assertTrue 'package A not installed' "[ ! -e '$tmpdir/testrootfs/a' ]"
}
testCleanup() {
luet cleanup --config $tmpdir/luet.yaml
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.gz' ]"
}
# Load shUnit2.
. "$ROOT_DIR/tests/integration/shunit2"/shunit2

8
tests/integration/run.sh
View File

@ -11,7 +11,7 @@ popd
export PATH=$ROOT_DIR/tests/integration/bin/:$PATH
"$ROOT_DIR/tests/integration/01_simple.sh"
"$ROOT_DIR/tests/integration/01_simple_gzip.sh"
"$ROOT_DIR/tests/integration/02_create_repo_from_config.sh"
for script in $(ls "$ROOT_DIR/tests/integration/" | grep '^[0-9]*_.*.sh'); do
echo "Executing script '$script'."
$ROOT_DIR/tests/integration/$script
done

5
vendor/github.com/crillab/gophersat/solver/card.go generated vendored
View File

@ -15,10 +15,11 @@ func AtLeast1(lits ...int) CardConstr {
// AtMost1 returns a cardinality constraint stating that at most one of the given lits can be true.
func AtMost1(lits ...int) CardConstr {
negated := make([]int, len(lits))
for i, lit := range lits {
lits[i] = -lit
negated[i] = -lit
}
return CardConstr{Lits: lits, AtLeast: len(lits) - 1}
return CardConstr{Lits: negated, AtLeast: len(lits) - 1}
}
// Exactly1 returns two cardinality constraints stating that exactly one of the given lits must be true.

10
vendor/github.com/crillab/gophersat/solver/parser.go generated vendored
View File

@ -55,17 +55,21 @@ func ParseSlice(cnf [][]int) *Problem {
return &pb
}
}
pb.simplify()
pb.simplify2()
return &pb
}
func isSpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}
// readInt reads an int from r.
// 'b' is the last read byte. It can be a space, a '-' or a digit.
// The int can be negated.
// All spaces before the int value are ignored.
// Can return EOF.
func readInt(b *byte, r *bufio.Reader) (res int, err error) {
for err == nil && (*b == ' ' || *b == '\t' || *b == '\n' || *b == '\r') {
for err == nil && isSpace(*b) {
*b, err = r.ReadByte()
}
if err == io.EOF {
@ -88,7 +92,7 @@ func readInt(b *byte, r *bufio.Reader) (res int, err error) {
}
res = 10*res + int(*b-'0')
*b, err = r.ReadByte()
if *b == ' ' || *b == '\t' || *b == '\n' || *b == '\r' {
if isSpace(*b) {
break
}
}

14
vendor/github.com/crillab/gophersat/solver/parser_pb.go generated vendored
View File

@ -65,6 +65,14 @@ func ParseCardConstrs(constrs []CardConstr) *Problem {
return &pb
}
func (pb *Problem) appendClause(constr PBConstr) {
lits := make([]Lit, len(constr.Lits))
for j, val := range constr.Lits {
lits[j] = IntToLit(int32(val))
}
pb.Clauses = append(pb.Clauses, NewPBClause(lits, constr.Weights, constr.AtLeast))
}
// ParsePBConstrs parses and returns a PB problem from PBConstr values.
func ParsePBConstrs(constrs []PBConstr) *Problem {
var pb Problem
@ -100,11 +108,7 @@ func ParsePBConstrs(constrs []PBConstr) *Problem {
}
}
} else {
lits := make([]Lit, len(constr.Lits))
for j, val := range constr.Lits {
lits[j] = IntToLit(int32(val))
}
pb.Clauses = append(pb.Clauses, NewPBClause(lits, constr.Weights, card))
pb.appendClause(constr)
}
}
pb.Model = make([]decLevel, pb.NbVars)

7
vendor/github.com/crillab/gophersat/solver/pb.go generated vendored
View File

@ -57,12 +57,17 @@ func GtEq(lits []int, weights []int, n int) PBConstr {
if len(weights) != 0 && len(lits) != len(weights) {
panic("not as many lits as weights")
}
for i := range weights {
for i := 0; i < len(weights); i++ {
if weights[i] < 0 {
weights[i] = -weights[i]
n += weights[i]
lits[i] = -lits[i]
}
if weights[i] == 0 {
weights = append(weights[:i], weights[i+1:]...)
lits = append(lits[:i], lits[i+1:]...)
i--
}
}
return PBConstr{Lits: lits, Weights: weights, AtLeast: n}
}

14
vendor/github.com/crillab/gophersat/solver/solver.go generated vendored
View File

@ -87,10 +87,16 @@ func New(problem *Problem) *Solver {
return &Solver{status: Unsat}
}
nbVars := problem.NbVars
trailCap := nbVars
if len(problem.Units) > trailCap {
trailCap = len(problem.Units)
}
s := &Solver{
nbVars: nbVars,
status: problem.Status,
trail: make([]Lit, len(problem.Units), nbVars),
trail: make([]Lit, len(problem.Units), trailCap),
model: problem.Model,
activity: make([]float64, nbVars),
polarity: make([]bool, nbVars),
@ -343,7 +349,7 @@ func (s *Solver) propagateAndSearch(lit Lit, lvl decLevel) Status {
return Indet
}
if s.Stats.NbConflicts >= s.wl.idxReduce*s.wl.nbMax {
s.wl.idxReduce = (s.Stats.NbConflicts / s.wl.nbMax) + 1
s.wl.idxReduce = s.Stats.NbConflicts/s.wl.nbMax + 1
s.reduceLearned()
s.bumpNbMax()
}
@ -738,7 +744,7 @@ func (s *Solver) Optimal(results chan Result, stop chan struct{}) (res Result) {
copy(s.lastModel, s.model) // Save this model: it might be the last one
cost = 0
for i, lit := range s.minLits {
if (s.model[lit.Var()] > 0) == lit.IsPositive() {
if s.model[lit.Var()] > 0 == lit.IsPositive() {
if s.minWeights == nil {
cost++
} else {
@ -803,7 +809,7 @@ func (s *Solver) Minimize() int {
copy(s.lastModel, s.model) // Save this model: it might be the last one
cost = 0
for i, lit := range s.minLits {
if (s.model[lit.Var()] > 0) == lit.IsPositive() {
if s.model[lit.Var()] > 0 == lit.IsPositive() {
if s.minWeights == nil {
cost++
} else {

24
vendor/github.com/ecooper/qlearning/.gitignore generated vendored Normal file
View File

@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

21
vendor/github.com/ecooper/qlearning/LICENSE generated vendored Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2016 Eric Cooper
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

72
vendor/github.com/ecooper/qlearning/README.md generated vendored Normal file
View File

@ -0,0 +1,72 @@
# qlearning
The qlearning package provides a series of interfaces and utilities to implement
the [Q-Learning](https://en.wikipedia.org/wiki/Q-learning) algorithm in
Go.
This project was largely inspired by [flappybird-qlearning-
bot](https://github.com/chncyhn/flappybird-qlearning-bot).
*Until a release is tagged, qlearning should be considered highly
experimental and mostly a fun toy.*
## Installation
```shell
$ go get https://github.com/ecooper/qlearning
```
## Quickstart
qlearning provides example implementations in the [examples](examples/)
directory of the project.
[hangman.go](examples/hangman.go) provides a naive implementation of
[Hangman](https://en.wikipedia.org/wiki/Hangman_(game)) for use with
qlearning.
```shell
$ cd $GOPATH/src/github.com/ecooper/qlearning/examples
$ go run hangman.go -h
Usage of hangman:
-debug
Set debug
-games int
Play N games (default 5000000)
-progress int
Print progress messages every N games (default 1000)
-wordlist string
Path to a wordlist (default "./wordlist.txt")
-words int
Use N words from wordlist (default 10000)
```
By default, running [hangman.go](examples/hangman.go) will play millions
of games against a 10,000-word corpus. That's a bit overkill for just
trying out qlearning. You can run it against a smaller number of words
for a few number of games using the `-games` and `-words` flags.
```shell
$ go run hangman.go -words 100 -progress 1000 -games 5000
100 words loaded
1000 games played: 92 WINS 908 LOSSES 9% WIN RATE
2000 games played: 447 WINS 1553 LOSSES 36% WIN RATE
3000 games played: 1064 WINS 1936 LOSSES 62% WIN RATE
4000 games played: 1913 WINS 2087 LOSSES 85% WIN RATE
5000 games played: 2845 WINS 2155 LOSSES 93% WIN RATE
Agent performance: 5000 games played, 2845 WINS 2155 LOSSES 57% WIN RATE
```
"WIN RATE" per progress report is isolated within that cycle, a group of
1000 games in this example. The win rate is meant to show the velocity
of learning by the agent. If it is "learning", the win rate should be
increasing until reaching convergence.
As you can see, after 5000 games, the agent is able to "learn" and play
hangman against a 100-word vocabulary.
## Usage
See [godocs](https://godoc.org/github.com/ecooper/qlearning) for the
package documentation.

167
vendor/github.com/ecooper/qlearning/qlearning.go generated vendored Normal file
View File

@ -0,0 +1,167 @@
// Package qlearning is an experimental set of interfaces and helpers to
// implement the Q-learning algorithm in Go.
//
// This is highly experimental and should be considered a toy.
//
// See https://github.com/ecooper/qlearning/tree/master/examples for
// implementation examples.
package qlearning
import (
"fmt"
"math/rand"
"time"
)
// State is an interface wrapping the current state of the model.
type State interface {
// String returns a string representation of the given state.
// Implementers should take care to insure that this is a consistent
// hash for a given state.
String() string
// Next provides a slice of possible Actions that could be applied to
// a state.
Next() []Action
}
// Action is an interface wrapping an action that can be applied to the
// model's current state.
//
// BUG (ecooper): A state should apply an action, not the other way
// around.
type Action interface {
String() string
Apply(State) State
}
// Rewarder is an interface wrapping the ability to provide a reward
// for the execution of an action in a given state.
type Rewarder interface {
// Reward calculates the reward value for a given action in a given
// state.
Reward(action *StateAction) float32
}
// Agent is an interface for a model's agent and is able to learn
// from actions and return the current Q-value of an action at a given state.
type Agent interface {
// Learn updates the model for a given state and action, using the
// provided Rewarder implementation.
Learn(*StateAction, Rewarder)
// Value returns the current Q-value for a State and Action.
Value(State, Action) float32
// Return a string representation of the Agent.
String() string
}
// StateAction is a struct grouping an action to a given State. Additionally,
// a Value can be associated to StateAction, which is typically the Q-value.
type StateAction struct {
State State
Action Action
Value float32
}
// NewStateAction creates a new StateAction for a State and Action.
func NewStateAction(state State, action Action, val float32) *StateAction {
return &StateAction{
State: state,
Action: action,
Value: val,
}
}
// Next uses an Agent and State to find the highest scored Action.
//
// In the case of Q-value ties for a set of actions, a random
// value is selected.
func Next(agent Agent, state State) *StateAction {
best := make([]*StateAction, 0)
bestVal := float32(0.0)
for _, action := range state.Next() {
val := agent.Value(state, action)
if bestVal == float32(0.0) {
best = append(best, NewStateAction(state, action, val))
bestVal = val
} else {
if val > bestVal {
best = []*StateAction{NewStateAction(state, action, val)}
bestVal = val
} else if val == bestVal {
best = append(best, NewStateAction(state, action, val))
}
}
}
return best[rand.Intn(len(best))]
}
// SimpleAgent is an Agent implementation that stores Q-values in a
// map of maps.
type SimpleAgent struct {
q map[string]map[string]float32
lr float32
d float32
}
// NewSimpleAgent creates a SimpleAgent with the provided learning rate
// and discount factor.
func NewSimpleAgent(lr, d float32) *SimpleAgent {
return &SimpleAgent{
q: make(map[string]map[string]float32),
d: d,
lr: lr,
}
}
// getActions returns the current Q-values for a given state.
func (agent *SimpleAgent) getActions(state string) map[string]float32 {
if _, ok := agent.q[state]; !ok {
agent.q[state] = make(map[string]float32)
}
return agent.q[state]
}
// Learn updates the existing Q-value for the given State and Action
// using the Rewarder.
//
// See https://en.wikipedia.org/wiki/Q-learning#Algorithm
func (agent *SimpleAgent) Learn(action *StateAction, reward Rewarder) {
current := action.State.String()
next := action.Action.Apply(action.State).String()
actions := agent.getActions(current)
maxNextVal := float32(0.0)
for _, v := range agent.getActions(next) {
if v > maxNextVal {
maxNextVal = v
}
}
currentVal := actions[action.Action.String()]
actions[action.Action.String()] = currentVal + agent.lr*(reward.Reward(action)+agent.d*maxNextVal-currentVal)
}
// Value gets the current Q-value for a State and Action.
func (agent *SimpleAgent) Value(state State, action Action) float32 {
return agent.getActions(state.String())[action.String()]
}
// String returns the current Q-value map as a printed string.
//
// BUG (ecooper): This is useless.
func (agent *SimpleAgent) String() string {
return fmt.Sprintf("%v", agent.q)
}
func init() {
rand.Seed(time.Now().UTC().UnixNano())
}

6
vendor/modules.txt vendored
View File

@ -49,7 +49,7 @@ github.com/containerd/continuity/pathdriver
github.com/containerd/continuity/syscallx
# github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d
github.com/cpuguy83/go-md2man/v2/md2man
# github.com/crillab/gophersat v1.1.7
# github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
github.com/crillab/gophersat/bf
github.com/crillab/gophersat/solver
# github.com/cyphar/filepath-securejoin v0.2.2
@ -93,6 +93,8 @@ github.com/docker/go-units
github.com/docker/libnetwork/ipamutils
# github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7
github.com/docker/libtrust
# github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
github.com/ecooper/qlearning
# github.com/fatih/color v1.7.0
github.com/fatih/color
# github.com/fsnotify/fsnotify v1.4.7
@ -263,8 +265,8 @@ go.uber.org/multierr
go.uber.org/tools/update-license
# go.uber.org/zap v1.13.0
go.uber.org/zap
go.uber.org/zap/internal/bufferpool
go.uber.org/zap/zapcore
go.uber.org/zap/internal/bufferpool
go.uber.org/zap/buffer
go.uber.org/zap/internal/color
go.uber.org/zap/internal/exit