Tag 0.9

Respect rootfs path for configs and url

.travis.yml

@@ -1,15 +1,31 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
   - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
   - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
   - sudo -E env "PATH=$PATH" make deps
 script:
   - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
 #after_success:
 #  - |
 #      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then

cmd/build.go

@@ -15,9 +15,11 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 
+	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
 	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/compiler/backend"
@@ -83,7 +85,13 @@ var buildCmd = &cobra.Command{
 		full, _ := cmd.Flags().GetBool("full")
 		skip, _ := cmd.Flags().GetBool("skip-if-metadata-exists")
 		concurrent, _ := cmd.Flags().GetBool("solver-concurrent")
+		var results Results
 
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			LuetCfg.GetLogging().SetLogLevel("error")
+		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
 		compilerSpecs := compiler.NewLuetCompilationspecs()
 		var compilerBackend compiler.CompilerBackend
 		var db pkg.PackageDatabase
@@ -205,9 +213,58 @@ var buildCmd = &cobra.Command{
 		if revdeps {
 			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []compiler.CompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				packs, err := luetCompiler.ComputeDepTree(sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range packs {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					Info(p.String())
+				}
+			}
 		} else {
 			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
-
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
@@ -252,5 +309,9 @@ func init() {
 	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
 	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
 
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+
 	RootCmd.AddCommand(buildCmd)
 }

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.8.11"
+	LuetCLIVersion = "0.9"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/search.go

@@ -32,6 +32,7 @@ type PackageResult struct {
 	Category   string `json:"category"`
 	Version    string `json:"version"`
 	Repository string `json:"repository"`
+	Target     string `json:"target"`
 	Hidden     bool   `json:"hidden"`
 }
 
@@ -39,6 +40,10 @@ type Results struct {
 	Packages []PackageResult `json:"packages"`
 }
 
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
 var searchCmd = &cobra.Command{
 	Use:     "search <term>",
 	Short:   "Search packages",

cmd/uninstall.go

@@ -75,6 +75,9 @@ var uninstallCmd = &cobra.Command{
 			}
 			Debug("Solver", LuetCfg.GetSolverOptions().CompactString())
 
+			// Load config protect configs
+			installer.LoadConfigProtectConfs(LuetCfg)
+
 			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
 				Concurrency:        LuetCfg.GetGeneral().Concurrency,
 				SolverOptions:      *LuetCfg.GetSolverOptions(),

cmd/upgrade.go

@@ -78,6 +78,9 @@ var upgradeCmd = &cobra.Command{
 
 		Debug("Solver", LuetCfg.GetSolverOptions().String())
 
+		// Load config protect configs
+		installer.LoadConfigProtectConfs(LuetCfg)
+
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
 			Concurrency:                LuetCfg.GetGeneral().Concurrency,
 			SolverOptions:              *LuetCfg.GetSolverOptions(),

contrib/config/get_luet_root.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=0.8.6
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+wget -q https://github.com/mudler/luet/releases/download/0.8.6/luet-0.8.6-linux-amd64 -O luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  wget -q  https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example -O /etc/luet/config.protect.d/01_etc.yml
+fi
+wget -q https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index/mocaccino-repository-index.yml -O /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install repository/luet repository/mocaccino-repository-index
+./luet install system/luet system/luet-extensions
+
+rm -rf luet

contrib/config/luet.yaml

@@ -69,6 +69,7 @@
 #   Default $TMPDIR/tmpluet
 #   tmpdir_base: "/tmp/tmpluet"
 #
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------
@@ -93,6 +94,11 @@
 # annotation.
 # config_protect_skip: false
 #
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/moby/sys/mount v0.1.1-0.20200320164225-6154f11e6840 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.12.1
 	github.com/onsi/gomega v1.10.0
 	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
@@ -33,7 +34,6 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.6.3
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
 	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect

go.sum

@@ -504,6 +504,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
+github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
+github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -699,8 +701,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
 github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
-github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b h1:wJSBFlabo96ySlmSX0a02WAPyGxagzTo9c5sk3sHP3E=
-github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b/go.mod h1:YIyOMT17IKD8FbLO8RfCJZd2qAZiOnIfuYePIeESwWc=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=

pkg/compiler/artifact.go

@@ -330,35 +330,25 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 
 func (a *PackageArtifact) GetProtectFiles() []string {
 	ans := []string{}
+	annotationDir := ""
 
-	if !LuetCfg.ConfigProtectSkip &&
-		LuetCfg.GetConfigProtectConfFiles() != nil &&
-		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+	if !LuetCfg.ConfigProtectSkip {
 
-		for _, file := range a.Files {
-			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
-				for _, dir := range conf.Directories {
-					// Note file is without / at begin.
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						// docker archive modifier works with path without / at begin.
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
 			}
-
-			if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
-				dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
-				if ok {
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
-			}
-
-		nextFile:
 		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
 	}
 
 	return ans
@@ -526,7 +516,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 }
 
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
+func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
 
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
@@ -556,7 +546,8 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 	}
 
 	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
+
+	if len(includes) > 0 && len(excludes) == 0 {
 		var includeRegexp []*regexp.Regexp
 		for _, i := range includes {
 			r, e := regexp.Compile(i)
@@ -584,6 +575,81 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 				Debug("File ", a.Name, " deleted")
 			}
 		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+
+		var includeRegexp []*regexp.Regexp
+		for _, i := range includes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			includeRegexp = append(includeRegexp, r)
+		}
+		var excludeRegexp []*regexp.Regexp
+		for _, i := range excludes {
+			r, e := regexp.Compile(i)
+			if e != nil {
+				Warning("Failed compiling regex:", e)
+				continue
+			}
+			excludeRegexp = append(excludeRegexp, r)
+		}
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
+		}
+
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {

pkg/compiler/artifact_test.go

@@ -112,21 +112,25 @@ RUN echo bar > /test2`))
 			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
 			Expect(err).ToNot(HaveOccurred())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(diffs).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, None)
+			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
 			err = helpers.Untar(artifact.GetPath(), unpacked, false)

pkg/compiler/backend/simpledocker_test.go

@@ -101,15 +101,19 @@ RUN echo bar > /test2`))
 			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
+			artifacts := []ArtifactNode{}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
 			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
 				[]ArtifactLayer{{
 					FromImage: filepath.Join(tmpdir2, "output1.tar"),
 					ToImage:   filepath.Join(tmpdir, "output2.tar"),
 					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+						Additions: artifacts,
 					},
 				}}))
 

pkg/compiler/compiler.go

@@ -181,7 +181,7 @@ func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpec
 	return artifacts, allErrors
 }
 
-func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string) error {
+func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, include bool) error {
 	var includeRegexp []*regexp.Regexp
 	for _, i := range includes {
 		r, e := regexp.Compile(i)
@@ -213,7 +213,7 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string
 			}
 		}
 
-		if !match {
+		if include && !match || !include && match {
 			toRemove = append(toRemove, currentpath)
 		}
 
@@ -421,7 +421,11 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 		if len(p.GetIncludes()) > 0 {
 			// strip from includes
-			cs.stripIncludesFromRootfs(p.GetIncludes(), rootfs)
+			cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
+		}
+		if len(p.GetExcludes()) > 0 {
+			// strip from includes
+			cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
@@ -438,7 +442,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate changes from layers")
 		}
-		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
+		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")
 		}

pkg/compiler/compiler_test.go

@@ -265,6 +265,146 @@ var _ = Describe("Compiler", func() {
 			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
 		})
 
+		It("Compiles and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes files", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludesincludes")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
+		})
+
+		It("Compiles and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
+		It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/excludeincludeimage")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			//		err = generalRecipe.Tree().ResolveDeps(3)
+			//		Expect(err).ToNot(HaveOccurred())
+
+			spec.SetOutputPath(tmpdir)
+			compiler.SetConcurrency(1)
+			artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+			Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
+		})
+
 		It("Compiles and includes ony wanted files also from unpacked packages", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
 			tmpdir, err := ioutil.TempDir("", "package")

pkg/compiler/interface.go

@@ -147,6 +147,7 @@ type ArtifactLayersSummary struct {
 type CompilationSpec interface {
 	ImageUnpack() bool // tells if the definition is just an image
 	GetIncludes() []string
+	GetExcludes() []string
 
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error

pkg/compiler/spec.go

@@ -102,6 +102,7 @@ type LuetCompilationSpec struct {
 	OutputPath string   `json:"-"` // Where the build processfiles go
 	Unpack     bool     `json:"unpack"`
 	Includes   []string `json:"includes"`
+	Excludes   []string `json:"excludes"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -148,6 +149,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
 	return cs.Includes
 }
 
+func (cs *LuetCompilationSpec) GetExcludes() []string {
+	return cs.Excludes
+}
+
 func (cs *LuetCompilationSpec) GetRetrieve() []string {
 	return cs.Retrieve
 }

pkg/config/config.go

@@ -97,7 +97,7 @@ type LuetSystemConfig struct {
 	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
-func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
 	dbpath = filepath.Join(dbpath, "repos/"+name)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -107,7 +107,7 @@ func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	dbpath := filepath.Join(sc.Rootfs,
 		sc.DatabasePath)
 	err := os.MkdirAll(dbpath, os.ModePerm)
@@ -117,7 +117,7 @@ func (sc LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
 	return dbpath
 }
 
-func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	var cachepath string
 	if sc.PkgsCachePath != "" {
 		cachepath = sc.PkgsCachePath
@@ -135,6 +135,10 @@ func (sc LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
 	return
 }
 
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
 type LuetRepository struct {
 	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
 	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
@@ -204,6 +208,7 @@ type LuetConfig struct {
 	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
 	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
 	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `mapstructure:"config_from_host"`
 	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
 	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
 
@@ -251,6 +256,8 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
 	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
 	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 

pkg/config/config_protect.go

@@ -18,6 +18,8 @@ package config
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 )
 
 type ConfigProtectConfFile struct {
@@ -39,3 +41,79 @@ func (c *ConfigProtectConfFile) String() string {
 	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
 		c.Directories)
 }
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/config/config_protect_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/helpers/file.go

@@ -24,6 +24,37 @@ import (
 	copy "github.com/otiai10/copy"
 )
 
+func OrderFiles(target string, files []string) ([]string, []string) {
+
+	var newFiles []string
+	var notPresent []string
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			notPresent = append(notPresent, f)
+			continue
+		}
+		if m := fi.Mode(); !m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	for _, f := range files {
+		target := filepath.Join(target, f)
+		fi, err := os.Lstat(target)
+		if err != nil {
+			continue
+		}
+		if m := fi.Mode(); m.IsDir() {
+			newFiles = append(newFiles, f)
+		}
+	}
+
+	return newFiles, notPresent
+}
+
 func ListDir(dir string) ([]string, error) {
 	content := []string{}
 

pkg/helpers/file_test.go

@@ -16,6 +16,10 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -28,4 +32,33 @@ var _ = Describe("Helpers", func() {
 			Expect(Exists("../../tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml.not.exists")).To(BeFalse())
 		})
 	})
+
+	Context("Orders dir and files correctly", func() {
+		It("puts files first and folders at end", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "bar"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "bar", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = os.MkdirAll(filepath.Join(testDir, "baz2"), 0755)
+			Expect(err).ToNot(HaveOccurred())
+			err = ioutil.WriteFile(filepath.Join(testDir, "baz2", "foo"), []byte("test\n"), 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			ordered, notExisting := OrderFiles(testDir, []string{"bar", "baz", "bar/foo", "baz2", "foo", "baz2/foo", "notexisting"})
+
+			Expect(ordered).To(Equal([]string{"baz", "bar/foo", "foo", "baz2/foo", "bar", "baz2"}))
+			Expect(notExisting).To(Equal([]string{"notexisting"}))
+		})
+	})
 })

pkg/installer/client/local.go

@@ -38,15 +38,26 @@ func NewLocalClient(r RepoData) *LocalClient {
 func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) {
 	var err error
 
+	rootfs := ""
 	artifactName := path.Base(artifact.GetPath())
 	cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
 
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Check if file is already in cache
 	if helpers.Exists(cacheFile) {
 		Info("Use artifact", artifactName, "from cache.")
 	} else {
 		ok := false
 		for _, uri := range c.RepoData.Urls {
+
+			uri = filepath.Join(rootfs, uri)
+
 			Info("Downloading artifact", artifactName, "from", uri)
 
 			//defer os.Remove(file.Name())
@@ -72,8 +83,20 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	var err error
 	var file *os.File = nil
 
+	rootfs := ""
+
+	if !config.LuetCfg.ConfigFromHost {
+		rootfs, err = config.LuetCfg.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return "", err
+		}
+	}
+
 	ok := false
 	for _, uri := range c.RepoData.Urls {
+
+		uri = filepath.Join(rootfs, uri)
+
 		Info("Downloading file", name, "from", uri)
 		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {

pkg/installer/config_protect.go

@@ -19,6 +19,7 @@ package installer
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -29,8 +30,21 @@ import (
 
 func LoadConfigProtectConfs(c *LuetConfig) error {
 	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
 		Debug("Parsing Config Protect Directory", cdir, "...")
 
 		files, err := ioutil.ReadDir(cdir)

pkg/installer/installer.go

@@ -359,30 +359,39 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 	var solution solver.PackagesAssertions
 
 	if !l.Options.NoDeps {
+		Info(":deciduous_tree: Computing installation, hang tight")
 		solv := solver.NewResolver(solver.Options{Type: l.Options.SolverOptions.Implementation, Concurrency: l.Options.Concurrency}, s.Database, allRepos, pkg.NewInMemoryDatabase(false), l.Options.SolverOptions.Resolver())
 		solution, err = solv.Install(p)
+		/// TODO: PackageAssertions needs to be a map[fingerprint]pack so lookup is in O(1)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed solving solution for package")
 		}
+		Info(":deciduous_tree: Finished calculating dependencies")
 		// Gathers things to install
+		Info(":deciduous_tree: Checking for packages already installed, and prepare for installation")
 		for _, assertion := range solution {
 			if assertion.Value {
+				if _, err := s.Database.FindPackage(assertion.Package); err == nil {
+					// skip matching if it is installed already
+					continue
+				}
 				packagesToInstall = append(packagesToInstall, assertion.Package)
 			}
 		}
 	} else if !l.Options.OnlyDeps {
 		for _, currentPack := range p {
+			if _, err := s.Database.FindPackage(currentPack); err == nil {
+				// skip matching if it is installed already
+				continue
+			}
 			packagesToInstall = append(packagesToInstall, currentPack)
 		}
 	}
-	Info(":deciduous_tree: Finding packages to install")
+	Info(":deciduous_tree: Finding packages to install from :cloud:")
 	// Gathers things to install
 	for _, currentPack := range packagesToInstall {
 		// Check if package is already installed.
-		if _, err := s.Database.FindPackage(currentPack); err == nil {
-			// skip matching if it is installed already
-			continue
-		}
+
 		matches := syncedRepos.PackageMatches(pkg.Packages{currentPack})
 		if len(matches) == 0 {
 			return errors.New("Failed matching solutions against repository for " + currentPack.HumanReadableString() + " where are definitions coming from?!")
@@ -601,16 +610,39 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 }
 
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
 	files, err := s.Database.GetPackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed getting installed files")
 	}
 
-	// Remove from target
-	for _, f := range files {
-		target := filepath.Join(s.Target, f)
-		Debug("Removing", target)
+	if !config.LuetCfg.ConfigProtectSkip {
 
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
+	toRemove, notPresent := helpers.OrderFiles(s.Target, files)
+
+	// Remove from target
+	for _, f := range toRemove {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		Debug("Removing", target)
 		if l.Options.PreserveSystemEssentialData &&
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {
@@ -618,11 +650,41 @@ func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
 			continue
 		}
 
-		err := os.Remove(target)
+		fi, err := os.Lstat(target)
 		if err != nil {
-			Warning("Failed removing file (not present in the system target ?)", target)
+			Warning("File not found (it was before?) ", err.Error())
+			continue
+		}
+		switch mode := fi.Mode(); {
+		case mode.IsDir():
+			files, err := ioutil.ReadDir(target)
+			if err != nil {
+				Warning("Failed reading folder", target, err.Error())
+			}
+			if len(files) != 0 {
+				Debug("Preserving not-empty folder", target)
+				continue
+			}
+		}
+
+		if err = os.Remove(target); err != nil {
+			Warning("Failed removing file (maybe not present in the system target anymore ?)", target, err.Error())
 		}
 	}
+
+	for _, f := range notPresent {
+		target := filepath.Join(s.Target, f)
+
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Preserving protected file:", f)
+			continue
+		}
+
+		if err = os.Remove(target); err != nil {
+			Debug("Failed removing file (not present in the system target)", target, err.Error())
+		}
+	}
+
 	err = s.Database.RemovePackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed removing package files from database")

pkg/package/database_boltdb.go

@@ -374,6 +374,11 @@ func (db *BoltDatabase) FindPackages(p Package) (Packages, error) {
 
 // FindPackageVersions return the list of the packages beloging to cat/name
 func (db *BoltDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	var versionsInWorld []Package
 	for _, w := range db.World() {
 		if w.GetName() != p.GetName() || w.GetCategory() != p.GetCategory() {

pkg/package/database_mem.go

@@ -225,6 +225,11 @@ func (db *InMemoryDatabase) FindPackage(p Package) (Package, error) {
 
 // FindPackages return the list of the packages beloging to cat/name
 func (db *InMemoryDatabase) FindPackageVersions(p Package) (Packages, error) {
+	// Provides: Treat as the replaced package here
+	if provided, err := db.getProvide(p); err == nil {
+		p = provided
+	}
+
 	versions, ok := db.CacheNoVersion[p.GetPackageName()]
 	if !ok {
 		return nil, errors.New("No versions found for package")

pkg/repository/loader.go

@@ -19,6 +19,7 @@ package repository
 import (
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"regexp"
 
 	"github.com/ghodss/yaml"
@@ -29,8 +30,21 @@ import (
 
 func LoadRepositories(c *LuetConfig) error {
 	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
 
 	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
 		Debug("Parsing Repository Directory", rdir, "...")
 
 		files, err := ioutil.ReadDir(rdir)

pkg/solver/benchmark_test.go

@@ -17,10 +17,12 @@ package solver_test
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strconv"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/tests/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -144,13 +146,17 @@ var _ = Describe("Solver Benchmarks", func() {
 	Context("Complex data sets - Parallel Upgrades", func() {
 		BeforeEach(func() {
 			db = pkg.NewInMemoryDatabase(false)
-			dbInstalled = pkg.NewInMemoryDatabase(false)
+			//	dbInstalled = pkg.NewInMemoryDatabase(false)
 			dbDefinitions = pkg.NewInMemoryDatabase(false)
 			s = NewSolver(Options{Type: ParallelSimple, Concurrency: 100}, dbInstalled, dbDefinitions, db)
 			if os.Getenv("BENCHMARK_TESTS") != "true" {
 				Skip("BENCHMARK_TESTS not enabled")
 			}
+			tmpfile, _ := ioutil.TempFile(os.TempDir(), "tests")
+			defer os.Remove(tmpfile.Name()) // clean up
+			dbInstalled = pkg.NewBoltDatabase(tmpfile.Name())
 		})
+
 		Measure("it should be fast in resolution from a 10000*8 dataset", func(b Benchmarker) {
 			runtime := b.Time("runtime", func() {
 				for i := 2; i < 10000; i++ {
@@ -206,6 +212,48 @@ var _ = Describe("Solver Benchmarks", func() {
 			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
 		}, 1)
 
+		Measure("it should be fast in installation with 12000 packages installed and 2000*8 available", func(b Benchmarker) {
+			runtime := b.Time("runtime", func() {
+				for i := 0; i < 2000; i++ {
+					C := pkg.NewPackage("C", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					E := pkg.NewPackage("E", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					F := pkg.NewPackage("F", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					G := pkg.NewPackage("G", strconv.Itoa(i), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+					H := pkg.NewPackage("H", strconv.Itoa(i), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+					D := pkg.NewPackage("D", strconv.Itoa(i), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+					B := pkg.NewPackage("B", strconv.Itoa(i), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+					A := pkg.NewPackage("A", strconv.Itoa(i), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+					for _, p := range []pkg.Package{A, B, C, D, E, F, G} {
+						_, err := dbDefinitions.CreatePackage(p)
+						Expect(err).ToNot(HaveOccurred())
+					}
+					fmt.Println("Creating package, run", i)
+				}
+
+				for i := 0; i < 12000; i++ {
+					x := helpers.RandomPackage()
+					_, err := dbInstalled.CreatePackage(x)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				G := pkg.NewPackage("G", strconv.Itoa(50000), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				H := pkg.NewPackage("H", strconv.Itoa(50000), []*pkg.DefaultPackage{G}, []*pkg.DefaultPackage{})
+				D := pkg.NewPackage("D", strconv.Itoa(50000), []*pkg.DefaultPackage{H}, []*pkg.DefaultPackage{})
+				B := pkg.NewPackage("B", strconv.Itoa(50000), []*pkg.DefaultPackage{D}, []*pkg.DefaultPackage{})
+				A := pkg.NewPackage("A", strconv.Itoa(50000), []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{})
+
+				ass, err := s.Install([]pkg.Package{A})
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(ass).To(ContainElement(PackageAssert{Package: pkg.NewPackage("A", "50000", []*pkg.DefaultPackage{B}, []*pkg.DefaultPackage{}), Value: true}))
+				//Expect(ass).To(Equal(5))
+				//	Expect(len(solution)).To(Equal(6))
+
+			})
+
+			Ω(runtime.Seconds()).Should(BeNumerically("<", 70), "Install() shouldn't take too long.")
+		}, 1)
+
 		PMeasure("it should be fast in resolution from a 50000 dataset with upgrade universe", func(b Benchmarker) {
 			runtime := b.Time("runtime", func() {
 				for i := 0; i < 2; i++ {

pkg/solver/decoder.go

@@ -22,9 +22,9 @@ import (
 	"unicode"
 
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/topsort"
 	toposort "github.com/philopon/go-toposort"
 	"github.com/pkg/errors"
-	"github.com/stevenle/topsort"
 )
 
 type PackagesAssertions []PackageAssert
@@ -181,7 +181,7 @@ func (assertions PackagesAssertions) Order(definitiondb pkg.PackageDatabase, fin
 			}
 			// Expand also here, as we need to order them (or instead the solver should give back the dep correctly?)
 			graph.AddEdge(currentPkg.GetFingerPrint(), requiredDef.GetFingerPrint())
-			added[requiredDef.GetFingerPrint()] = nil
+			added[requiredDef.GetFingerPrint()] = true
 		}
 	}
 	result, err := graph.TopSort(fingerprint)

pkg/solver/decoder_test.go

@@ -388,5 +388,37 @@ var _ = Describe("Decoder", func() {
 
 			Expect(solution4.Drop(Y).AssertionHash()).To(Equal(solution4.HashFrom(Y)))
 		})
+		for index := 0; index < 300; index++ { // Just to make sure we don't have false positives
+
+			It("Always same solution", func() {
+
+				X := pkg.NewPackage("X", "", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+				Y := pkg.NewPackage("Y", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				Z := pkg.NewPackage("Z", "", []*pkg.DefaultPackage{X}, []*pkg.DefaultPackage{})
+				W := pkg.NewPackage("W", "", []*pkg.DefaultPackage{Z, Y}, []*pkg.DefaultPackage{})
+
+				for _, p := range []pkg.Package{X, Y, Z} {
+					_, err := dbDefinitions.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				for _, p := range []pkg.Package{} {
+					_, err := dbInstalled.CreatePackage(p)
+					Expect(err).ToNot(HaveOccurred())
+				}
+
+				solution, err := s.Install([]pkg.Package{W})
+				Expect(err).ToNot(HaveOccurred())
+
+				orderW, err := solution.Order(dbDefinitions, W.GetFingerPrint())
+				Expect(err).ToNot(HaveOccurred())
+				Expect(orderW[0].Package.GetName()).To(Equal("X"))
+				Expect(orderW[1].Package.GetName()).To(Equal("Y"))
+				Expect(orderW[2].Package.GetName()).To(Equal("Z"))
+				Expect(orderW[3].Package.GetName()).To(Equal("W"))
+			})
+
+		}
+
 	})
 })

pkg/solver/parallel.go

@@ -556,10 +556,11 @@ func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAss
 
 	toUninstall := pkg.Packages{}
 	toInstall := pkg.Packages{}
-	availableCache := map[string]pkg.Packages{}
+
+	// we do this in memory so we take into account of provides
+	universe := pkg.NewInMemoryDatabase(false)
 	for _, p := range s.DefinitionDatabase.World() {
-		// Each one, should be expanded
-		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
+		universe.CreatePackage(p)
 	}
 
 	installedcopy := pkg.NewInMemoryDatabase(false)
@@ -575,10 +576,10 @@ func (s *Parallel) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAss
 			defer wg.Done()
 			for p := range c {
 				installedcopy.CreatePackage(p)
-				packages, ok := availableCache[p.GetName()+p.GetCategory()]
-				if ok && len(packages) != 0 {
+				packages, err := universe.FindPackageVersions(p)
+				if err == nil && len(packages) != 0 {
 					best := packages.Best(nil)
-					if best.GetVersion() != p.GetVersion() {
+					if !best.Matches(p) {
 						results <- []pkg.Package{p, best}
 					}
 				}

pkg/solver/solver.go

@@ -489,20 +489,20 @@ func (s *Solver) Upgrade(checkconflicts, full bool) (pkg.Packages, PackagesAsser
 	toUninstall := pkg.Packages{}
 	toInstall := pkg.Packages{}
 
-	availableCache := map[string]pkg.Packages{}
+	// we do this in memory so we take into account of provides
+	universe := pkg.NewInMemoryDatabase(false)
 	for _, p := range s.DefinitionDatabase.World() {
-		// Each one, should be expanded
-		availableCache[p.GetName()+p.GetCategory()] = append(availableCache[p.GetName()+p.GetCategory()], p)
+		universe.CreatePackage(p)
 	}
 
 	installedcopy := pkg.NewInMemoryDatabase(false)
 
 	for _, p := range s.InstalledDatabase.World() {
 		installedcopy.CreatePackage(p)
-		packages, ok := availableCache[p.GetName()+p.GetCategory()]
-		if ok && len(packages) != 0 {
+		packages, err := universe.FindPackageVersions(p)
+		if err == nil && len(packages) != 0 {
 			best := packages.Best(nil)
-			if best.GetVersion() != p.GetVersion() {
+			if !best.Matches(p) {
 				toUninstall = append(toUninstall, p)
 				toInstall = append(toInstall, best)
 			}

pkg/solver/solver_test.go

@@ -1209,7 +1209,6 @@ var _ = Describe("Solver", func() {
 		})
 	})
 	Context("Upgrades", func() {
-
 		C := pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		C.SetCategory("test")
 		B := pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
@@ -1219,6 +1218,17 @@ var _ = Describe("Solver", func() {
 		A1 := pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
 		A1.SetCategory("test")
 
+		BeforeEach(func() {
+			C = pkg.NewPackage("c", "1.5", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			C.SetCategory("test")
+			B = pkg.NewPackage("b", "1.0", []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+			B.SetCategory("test")
+			A = pkg.NewPackage("a", "1.1", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A.SetCategory("test")
+			A1 = pkg.NewPackage("a", "1.2", []*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "b", Version: "1.0", Category: "test"}}, []*pkg.DefaultPackage{})
+			A1.SetCategory("test")
+		})
+
 		It("upgrades correctly", func() {
 			for _, p := range []pkg.Package{A1, B, C} {
 				_, err := dbDefinitions.CreatePackage(p)
@@ -1240,7 +1250,31 @@ var _ = Describe("Solver", func() {
 			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
 			Expect(solution).To(ContainElement(PackageAssert{Package: C, Value: false}))
 			Expect(len(solution)).To(Equal(3))
+		})
 
+		It("upgrades correctly with provides", func() {
+			B.SetProvides([]*pkg.DefaultPackage{&pkg.DefaultPackage{Name: "a", Version: ">=0", Category: "test"}, &pkg.DefaultPackage{Name: "c", Version: ">=0", Category: "test"}})
+
+			for _, p := range []pkg.Package{A1, B} {
+				_, err := dbDefinitions.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			for _, p := range []pkg.Package{A, C} {
+				_, err := dbInstalled.CreatePackage(p)
+				Expect(err).ToNot(HaveOccurred())
+			}
+			uninstall, solution, err := s.Upgrade(true, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(uninstall)).To(Equal(2))
+			Expect(uninstall[1].GetName()).To(Equal("c"))
+			Expect(uninstall[1].GetVersion()).To(Equal("1.5"))
+			Expect(uninstall[0].GetName()).To(Equal("a"))
+			Expect(uninstall[0].GetVersion()).To(Equal("1.1"))
+
+			Expect(solution).To(ContainElement(PackageAssert{Package: B, Value: true}))
+			Expect(len(solution)).To(Equal(1))
 		})
 
 		It("UpgradeUniverse upgrades correctly", func() {

tests/fixtures/excludeimage/build.yaml

@@ -0,0 +1,14 @@
+requires:
+- category: "layer"
+  name: "seed"
+  version: "1.0"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - echo artifact43 > /marvin
+unpack: true
+excludes:
+- marvin

tests/fixtures/excludeimage/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/excludeimage/seed/build.yaml

@@ -0,0 +1,2 @@
+image: alpine
+unpack: true

tests/fixtures/excludeimage/seed/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "seed"
+version: "1.0"

tests/fixtures/excludeincludeimage/build.yaml

@@ -0,0 +1,17 @@
+requires:
+- category: "layer"
+  name: "seed"
+  version: "1.0"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - echo artifact43 > /marvin
+unpack: true
+excludes:
+- marvin
+includes:
+- test.*
+- mar.*

tests/fixtures/excludeincludeimage/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/excludeincludeimage/seed/build.yaml

@@ -0,0 +1,2 @@
+image: alpine
+unpack: true

tests/fixtures/excludeincludeimage/seed/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "seed"
+version: "1.0"

tests/fixtures/excludes/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - echo artifact43 > /marvin
+  - echo "foo" > /marvot
+excludes:
+- marvot

tests/fixtures/excludes/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/fixtures/excludesincludes/build.yaml

@@ -0,0 +1,14 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /test5
+  - echo artifact6 > /test6
+  - echo artifact43 > /marvin
+  - echo "foo" > /marvot
+excludes:
+- marvot
+includes:
+- /test5
+- mar.*	

tests/fixtures/excludesincludes/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.0"

tests/helpers/package.go

@@ -0,0 +1,31 @@
+package helpers
+
+import (
+	"math/rand"
+	"strconv"
+	"time"
+
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+const charset = "abcdefghijklmnopqrstuvwxyz" +
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+var seededRand *rand.Rand = rand.New(
+	rand.NewSource(time.Now().UnixNano()))
+
+func StringWithCharset(length int, charset string) string {
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
+func String(length int) string {
+	return StringWithCharset(length, charset)
+}
+
+func RandomPackage() pkg.Package {
+	return pkg.NewPackage(String(5), strconv.Itoa(rand.Intn(100)), []*pkg.DefaultPackage{}, []*pkg.DefaultPackage{})
+}

tests/integration/01_simple.sh

@@ -43,6 +43,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/01_simple_gzip.sh

@@ -48,6 +48,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/01_simple_meta_gzip.sh

@@ -51,6 +51,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/02_create_repo_from_config.sh

@@ -28,6 +28,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/03_qlearning.sh

@@ -43,6 +43,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/04_retrieve.sh

@@ -43,6 +43,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/05_upgrade.sh

@@ -71,6 +71,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/06_search.sh

@@ -70,6 +70,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/07_finalizer.sh

@@ -42,6 +42,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/07_finalizer_custom.sh

@@ -42,6 +42,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/08_versioning.sh

@@ -50,6 +50,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/09_docker.sh

@@ -42,6 +42,7 @@ system:
   rootfs: /
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/10_tmpdir_cleanup.sh

@@ -44,6 +44,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/11_upgrade_universe.sh

@@ -71,6 +71,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/12_config_protect.sh

@@ -11,18 +11,19 @@ oneTimeTearDown() {
 }
 
 testBuild() {
-    mkdir $tmpdir/testbuild
-    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" --destination $tmpdir/testbuild --compression gzip test/a
+    mkdir $tmpdir/testrootfs/testbuild -p
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+      --destination $tmpdir/testrootfs/testbuild --compression gzip test/a
     buildst=$?
     assertEquals 'builds successfully' "$buildst" "0"
-    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testrootfs/testbuild/a-test-1.0.package.tar.gz' ]"
 }
 
 testRepo() {
     assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
     luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
-    --output $tmpdir/testbuild \
-    --packages $tmpdir/testbuild \
+    --output $tmpdir/testrootfs/testbuild \
+    --packages $tmpdir/testrootfs/testbuild \
     --name "test" \
     --descr "Test Repo" \
     --urls $tmpdir/testrootfs \
@@ -30,15 +31,14 @@ testRepo() {
 
     createst=$?
     assertEquals 'create repo successfully' "$createst" "0"
-    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+    assertTrue 'create repository' "[ -e '$tmpdir/testrootfs/testbuild/repository.yaml' ]"
 }
 
 testConfig() {
-    mkdir $tmpdir/testrootfs
 
-    mkdir $tmpdir/config.protect.d
+    mkdir $tmpdir/testrootfs/etc/luet/config.protect.d -p
 
-    cat <<EOF > $tmpdir/config.protect.d/conf1.yml
+    cat <<EOF > $tmpdir/testrootfs/etc/luet/config.protect.d/conf1.yml
 name: "protect1"
 dirs:
 - /etc/
@@ -52,13 +52,14 @@ system:
   database_path: "/"
   database_engine: "boltdb"
 config_protect_confdir:
-    - $tmpdir/config.protect.d
+    - /etc/luet/config.protect.d
+config_from_host: false
 repositories:
    - name: "main"
      type: "disk"
      enable: true
      urls:
-       - "$tmpdir/testbuild"
+       - "/testbuild"
 EOF
     luet config --config $tmpdir/luet.yaml
     res=$?
@@ -91,6 +92,7 @@ testUnInstall() {
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
     # TODO: we need remove it or not??
     assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
 }
 
 

tests/integration/13_config_protect_annotation.sh

@@ -54,6 +54,7 @@ system:
   database_engine: "boltdb"
 config_protect_confdir:
     - $tmpdir/config.protect.d
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"
@@ -91,6 +92,7 @@ testUnInstall() {
     assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
     # TODO: we need remove it or not??
     assertTrue 'config protect created' "[ -e '$tmpdir/testrootfs/opt/etc/._cfg0001_conf' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/opt/etc/conf' ]"
 }
 
 

tests/integration/14_upgrade_revision.sh

@@ -61,6 +61,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/15_symlinks.sh

@@ -42,6 +42,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/16_caps.sh

@@ -42,6 +42,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

tests/integration/17_config_protect_skip.sh

@@ -52,6 +52,7 @@ system:
   database_path: "/"
   database_engine: "boltdb"
 config_protect_skip: true
+config_from_host: true
 config_protect_confdir:
     - $tmpdir/config.protect.d
 repositories:

tests/integration/18_database.sh

@@ -43,6 +43,7 @@ system:
   rootfs: $tmpdir/testrootfs
   database_path: "/"
   database_engine: "boltdb"
+config_from_host: true
 repositories:
    - name: "main"
      type: "disk"

vendor/github.com/crillab/gophersat/bf/bf.go

@@ -230,6 +230,8 @@ func (a and) String() string {
 }
 
 func (a and) Eval(model map[string]bool) (res bool) {
+	res = true
+
 	for i, s := range a {
 		b := s.Eval(model)
 		if i == 0 {

vendor/github.com/crillab/gophersat/solver/learn.go

@@ -29,7 +29,7 @@ func (s *Solver) addClauseLits(confl *Clause, lvl decLevel, met, metLvl []bool,
 		if abs(s.model[v]) == lvl {
 			metLvl[v] = true
 			nbLvl++
-		} else if abs(s.model[v]) != 1 {
+		} else {
 			*lits = append(*lits, l)
 		}
 	}
@@ -39,8 +39,9 @@ func (s *Solver) addClauseLits(confl *Clause, lvl decLevel, met, metLvl []bool,
 var bufLits = make([]Lit, 10000) // Buffer for lits in learnClause. Used to reduce allocations.
 
 // learnClause creates a conflict clause and returns either:
-// the clause itself, if its len is at least 2.
-// a nil clause and a unit literal, if its len is exactly 1.
+// - the clause itself, if its len is at least 2,
+// - a nil clause and a unit literal, if its len is exactly 1,
+// - a nil clause and -1, if the empty clause was learned.
 func (s *Solver) learnClause(confl *Clause, lvl decLevel) (learned *Clause, unit Lit) {
 	s.clauseBumpActivity(confl)
 	lits := bufLits[:1]             // Not 0: make room for asserting literal
@@ -58,6 +59,10 @@ func (s *Solver) learnClause(confl *Clause, lvl decLevel) (learned *Clause, unit
 			ptr--
 		}
 		v := s.trail[ptr].Var()
+		if s.assumptions[v] {
+			// Now we only have assumed lits: this is a top-level conflict
+			return nil, -1
+		}
 		ptr--
 		nbLvl--
 		if reason := s.reason[v]; reason != nil {
@@ -65,7 +70,7 @@ func (s *Solver) learnClause(confl *Clause, lvl decLevel) (learned *Clause, unit
 			for i := 0; i < reason.Len(); i++ {
 				lit := reason.Get(i)
 				if v2 := lit.Var(); !met[v2] {
-					if s.litStatus(lit) != Unsat {
+					if s.litStatus(lit) != Unsat { // In clauses where cardinality > 1, some lits might be true in the conflict clause: ignore them
 						continue
 					}
 					met[v2] = true
@@ -73,7 +78,7 @@ func (s *Solver) learnClause(confl *Clause, lvl decLevel) (learned *Clause, unit
 					if abs(s.model[v2]) == lvl {
 						metLvl[v2] = true
 						nbLvl++
-					} else if abs(s.model[v2]) != 1 {
+					} else {
 						lits = append(lits, lit)
 					}
 				}
@@ -109,7 +114,7 @@ func (s *Solver) minimizeLearned(met []bool, learned []Lit) int {
 		} else {
 			for k := 0; k < reason.Len(); k++ {
 				lit := reason.Get(k)
-				if !met[lit.Var()] && abs(s.model[lit.Var()]) > 1 {
+				if !met[lit.Var()] /*&& abs(s.model[lit.Var()]) > 1*/ {
 					learned[sz] = learned[i]
 					sz++
 					break

vendor/github.com/crillab/gophersat/solver/parser.go

@@ -12,11 +12,25 @@ import (
 // The argument is supposed to be a well-formed CNF.
 func ParseSlice(cnf [][]int) *Problem {
 	var pb Problem
+	pb.parseSlice(cnf)
+	return &pb
+}
+
+// ParseSliceNb parse a slice of slice of lits and returns the equivalent problem.
+// The argument is supposed to be a well-formed CNF.
+// The number of vars is provided because clauses might be added to it later.
+func ParseSliceNb(cnf [][]int, nbVars int) *Problem {
+	pb := Problem{NbVars: nbVars}
+	pb.parseSlice(cnf)
+	return &pb
+}
+
+func (pb *Problem) parseSlice(cnf [][]int) {
 	for _, line := range cnf {
 		switch len(line) {
 		case 0:
 			pb.Status = Unsat
-			return &pb
+			return
 		case 1:
 			if line[0] == 0 {
 				panic("null unit clause")
@@ -31,7 +45,7 @@ func ParseSlice(cnf [][]int) *Problem {
 			lits := make([]Lit, len(line))
 			for j, val := range line {
 				if val == 0 {
-					panic("null literal in clause %q")
+					panic(fmt.Sprintf("null literal in clause %v", lits))
 				}
 				lits[j] = IntToLit(int32(val))
 				if v := int(lits[j].Var()); v >= pb.NbVars {
@@ -52,11 +66,10 @@ func ParseSlice(cnf [][]int) *Problem {
 			}
 		} else if pb.Model[v] > 0 != unit.IsPositive() {
 			pb.Status = Unsat
-			return &pb
+			return
 		}
 	}
 	pb.simplify2()
-	return &pb
 }
 
 func isSpace(b byte) bool {

vendor/github.com/crillab/gophersat/solver/problem.go

@@ -97,26 +97,6 @@ func (pb *Problem) updateStatus(nbClauses int) {
 	}
 }
 
-func (pb *Problem) simplify() {
-	idxClauses := make([][]int, pb.NbVars*2) // For each lit, indexes of clauses it appears in
-	removed := make([]bool, len(pb.Clauses)) // Clauses that have to be removed
-	for i := range pb.Clauses {
-		pb.simplifyClause(i, idxClauses, removed)
-		if pb.Status == Unsat {
-			return
-		}
-	}
-	for i := 0; i < len(pb.Units); i++ {
-		lit := pb.Units[i]
-		neg := lit.Negation()
-		clauses := idxClauses[neg]
-		for j := range clauses {
-			pb.simplifyClause(j, idxClauses, removed)
-		}
-	}
-	pb.rmClauses(removed)
-}
-
 func (pb *Problem) simplifyClause(idx int, idxClauses [][]int, removed []bool) {
 	c := pb.Clauses[idx]
 	k := 0

vendor/github.com/crillab/gophersat/solver/solver.go

@@ -54,15 +54,18 @@ func (m Model) String() string {
 
 // A Solver solves a given problem. It is the main data structure.
 type Solver struct {
-	Verbose   bool // Indicates whether the solver should display information during solving or not. False by default
-	nbVars    int
-	status    Status
-	wl        watcherList
-	trail     []Lit     // Current assignment stack
-	model     Model     // 0 means unbound, other value is a binding
-	lastModel Model     // Placeholder for last model found, useful when looking for several models
-	activity  []float64 // How often each var is involved in conflicts
-	polarity  []bool    // Preferred sign for each var
+	Verbose     bool        // Indicates whether the solver should display information during solving or not. False by default
+	Certified   bool        // Indicates whether a certificate should be generated during solving or not, using the RUP notation. This is useful to prove UNSAT instances. False by default.
+	CertChan    chan string // Indicates where to write the certificate. If Certified is true but CertChan is nil, the certificate will be written on stdout.
+	nbVars      int
+	status      Status
+	wl          watcherList
+	trail       []Lit     // Current assignment stack
+	model       Model     // 0 means unbound, other value is a binding
+	lastModel   Model     // Placeholder for last model found, useful when looking for several models
+	activity    []float64 // How often each var is involved in conflicts
+	polarity    []bool    // Preferred sign for each var
+	assumptions []bool    // True iff the var's binding is assumed
 	// For each var, clause considered when it was unified
 	// If the var is not bound yet, or if it was bound by a decision, value is nil.
 	reason          []*Clause
@@ -73,7 +76,7 @@ type Solver struct {
 	Stats           Stats   // Statistics about the solving process.
 	minLits         []Lit   // Lits to minimize if the problem was an optimization problem.
 	minWeights      []int   // Weight of each lit to minimize if the problem was an optimization problem.
-	asumptions      []Lit   // Literals that are, ideally, true. Useful when trying to minimize a function.
+	hypothesis      []Lit   // Literals that are, ideally, true. Useful when trying to minimize a function.
 	localNbRestarts int     // How many restarts since Solve() was called?
 	varDecay        float64 // On each var decay, how much the varInc should be decayed
 	trailBuf        []int   // A buffer while cleaning bindings
@@ -94,19 +97,20 @@ func New(problem *Problem) *Solver {
 	}
 
 	s := &Solver{
-		nbVars:     nbVars,
-		status:     problem.Status,
-		trail:      make([]Lit, len(problem.Units), trailCap),
-		model:      problem.Model,
-		activity:   make([]float64, nbVars),
-		polarity:   make([]bool, nbVars),
-		reason:     make([]*Clause, nbVars),
-		varInc:     1.0,
-		clauseInc:  1.0,
-		minLits:    problem.minLits,
-		minWeights: problem.minWeights,
-		varDecay:   defaultVarDecay,
-		trailBuf:   make([]int, nbVars),
+		nbVars:      nbVars,
+		status:      problem.Status,
+		trail:       make([]Lit, len(problem.Units), trailCap),
+		model:       problem.Model,
+		activity:    make([]float64, nbVars),
+		polarity:    make([]bool, nbVars),
+		assumptions: make([]bool, nbVars),
+		reason:      make([]*Clause, nbVars),
+		varInc:      1.0,
+		clauseInc:   1.0,
+		minLits:     problem.minLits,
+		minWeights:  problem.minWeights,
+		varDecay:    defaultVarDecay,
+		trailBuf:    make([]int, nbVars),
 	}
 	s.resetOptimPolarity()
 	s.initOptimActivity()
@@ -322,26 +326,11 @@ func (s *Solver) rebuildOrderHeap() {
 	s.varQueue.build(ints)
 }
 
-// satClause returns true iff c is satisfied by a literal assigned at top level.
-func (s *Solver) satClause(c *Clause) bool {
-	if c.Len() == 2 || c.Cardinality() != 1 || c.PseudoBoolean() {
-		// TODO improve this, but it will be ok since we only call this function for removing useless clauses.
-		return false
-	}
-	for i := 0; i < c.Len(); i++ {
-		lit := c.Get(i)
-		assign := s.model[lit.Var()]
-		if assign == 1 && lit.IsPositive() || assign == -1 && !lit.IsPositive() {
-			return true
-		}
-	}
-	return false
-}
-
 // propagate binds the given lit, propagates it and searches for a solution,
 // until it is found or a restart is needed.
 func (s *Solver) propagateAndSearch(lit Lit, lvl decLevel) Status {
 	for lit != -1 {
+		// log.Printf("picked %d at lvl %d", lit.Int(), lvl)
 		if conflict := s.unifyLiteral(lit, lvl); conflict == nil { // Pick new branch or restart
 			if s.lbdStats.mustRestart() {
 				s.lbdStats.clear()
@@ -363,15 +352,17 @@ func (s *Solver) propagateAndSearch(lit Lit, lvl decLevel) Status {
 			s.lbdStats.addConflict(len(s.trail))
 			learnt, unit := s.learnClause(conflict, lvl)
 			if learnt == nil { // Unit clause was learned: this lit is known for sure
+				if unit == -1 || (abs(s.model[unit.Var()]) == 1 && s.litStatus(unit) == Unsat) { // Top-level conflict
+					return s.setUnsat()
+				}
 				s.Stats.NbUnitLearned++
 				s.lbdStats.addLbd(1)
 				s.cleanupBindings(1)
+				s.addLearnedUnit(unit)
 				s.model[unit.Var()] = lvlToSignedLvl(unit, 1)
 				if conflict = s.unifyLiteral(unit, 1); conflict != nil { // top-level conflict
-					s.status = Unsat
-					return Unsat
+					return s.setUnsat()
 				}
-				// s.rmSatClauses()
 				s.rebuildOrderHeap()
 				lit = s.chooseLit()
 				lvl = 2
@@ -392,6 +383,19 @@ func (s *Solver) propagateAndSearch(lit Lit, lvl decLevel) Status {
 	return Sat
 }
 
+// Sets the status to unsat and do cleanup tasks.
+func (s *Solver) setUnsat() Status {
+	if s.Certified {
+		if s.CertChan == nil {
+			fmt.Printf("0\n")
+		} else {
+			s.CertChan <- "0"
+		}
+	}
+	s.status = Unsat
+	return Unsat
+}
+
 // Searches until a restart is needed.
 func (s *Solver) search() Status {
 	s.localNbRestarts++
@@ -455,6 +459,28 @@ func (s *Solver) Solve() Status {
 	return s.status
 }
 
+// Assume adds unit literals to the solver.
+// This is useful when calling the solver several times, e.g to keep it "hot" while removing clauses.
+func (s *Solver) Assume(lits []Lit) Status {
+	s.cleanupBindings(0)
+	s.trail = s.trail[:0]
+	s.assumptions = make([]bool, s.nbVars)
+
+	for _, lit := range lits {
+
+		s.addLearnedUnit(lit)
+		s.assumptions[lit.Var()] = true
+		s.trail = append(s.trail, lit)
+	}
+	s.status = Indet
+	if confl := s.propagate(0, 1); confl != nil {
+		// Conflict after unit propagation
+		s.status = Unsat
+		return s.status
+	}
+	return s.status
+}
+
 // Enumerate returns the total number of models for the given problems.
 // if "models" is non-nil, it will write models on it as soon as it discovers them.
 // models will be closed at the end of the method.
@@ -474,10 +500,11 @@ func (s *Solver) Enumerate(models chan []bool, stop chan struct{}) int {
 			}
 		}
 		if s.status == Sat {
-			nb++
+			copy(s.lastModel, s.model)
 			if models != nil {
-				copy(s.lastModel, s.model)
-				models <- s.Model()
+				nb += s.addCurrentModels(models)
+			} else {
+				nb += s.countCurrentModels()
 			}
 			s.status = Indet
 			lits := s.decisionLits()
@@ -543,7 +570,8 @@ func (s *Solver) CountModels() int {
 			}
 		}
 		if s.status == Sat {
-			nb++
+			s.lastModel = s.model
+			nb += s.countCurrentModels()
 			if s.Verbose {
 				fmt.Printf("c found %d model(s)\n", nb)
 			}
@@ -695,6 +723,52 @@ func (s *Solver) Model() []bool {
 	return res
 }
 
+// addCurrentModels is called when a model was found.
+// It returns the total number of models from this point, and sends all models on ch.
+// The number can be different of 1 if there are unbound variables.
+// For instance, if there are 4 variables in the problem and only 1, 3 and 4 are bound,
+// there are actually 2 models currently: one with 2 set to true, the other with 2 set to false.
+func (s *Solver) addCurrentModels(ch chan []bool) int {
+	unbound := make([]int, 0, s.nbVars) // indices of unbound variables
+	var nb uint64 = 1                   // total number of models found
+	model := make([]bool, s.nbVars)     // partial model
+	for i, lvl := range s.lastModel {
+		if lvl == 0 {
+			unbound = append(unbound, i)
+			nb *= 2
+		} else {
+			model[i] = lvl > 0
+		}
+	}
+	for i := uint64(0); i < nb; i++ {
+		for j := range unbound {
+			mask := uint64(1 << j)
+			cur := i & mask
+			idx := unbound[j]
+			model[idx] = cur != 0
+		}
+		model2 := make([]bool, len(model))
+		copy(model2, model)
+		ch <- model2
+	}
+	return int(nb)
+}
+
+// countCurrentModels is called when a model was found.
+// It returns the total number of models from this point.
+// The number can be different of 1 if there are unbound variables.
+// For instance, if there are 4 variables in the problem and only 1, 3 and 4 are bound,
+// there are actually 2 models currently: one with 2 set to true, the other with 2 set to false.
+func (s *Solver) countCurrentModels() int {
+	var nb uint64 = 1 // total number of models found
+	for _, lvl := range s.lastModel {
+		if lvl == 0 {
+			nb *= 2
+		}
+	}
+	return int(nb)
+}
+
 // Optimal returns the optimal solution, if any.
 // If results is non-nil, all solutions will be written to it.
 // In any case, results will be closed at the end of the call.
@@ -731,13 +805,13 @@ func (s *Solver) Optimal(results chan Result, stop chan struct{}) (res Result) {
 			maxCost += w
 		}
 	}
-	s.asumptions = make([]Lit, len(s.minLits))
+	s.hypothesis = make([]Lit, len(s.minLits))
 	for i, lit := range s.minLits {
-		s.asumptions[i] = lit.Negation()
+		s.hypothesis[i] = lit.Negation()
 	}
 	weights := make([]int, len(s.minWeights))
 	copy(weights, s.minWeights)
-	sort.Sort(wLits{lits: s.asumptions, weights: weights})
+	sort.Sort(wLits{lits: s.hypothesis, weights: weights})
 	s.lastModel = make(Model, len(s.model))
 	var cost int
 	for status == Sat {
@@ -766,7 +840,7 @@ func (s *Solver) Optimal(results chan Result, stop chan struct{}) (res Result) {
 		// Add a constraint incrementing current best cost
 		lits2 := make([]Lit, len(s.minLits))
 		weights2 := make([]int, len(s.minWeights))
-		copy(lits2, s.asumptions)
+		copy(lits2, s.hypothesis)
 		copy(weights2, weights)
 		s.AppendClause(NewPBClause(lits2, weights2, maxCost-cost+1))
 		s.rebuildOrderHeap()
@@ -796,13 +870,13 @@ func (s *Solver) Minimize() int {
 			maxCost += w
 		}
 	}
-	s.asumptions = make([]Lit, len(s.minLits))
+	s.hypothesis = make([]Lit, len(s.minLits))
 	for i, lit := range s.minLits {
-		s.asumptions[i] = lit.Negation()
+		s.hypothesis[i] = lit.Negation()
 	}
 	weights := make([]int, len(s.minWeights))
 	copy(weights, s.minWeights)
-	sort.Sort(wLits{lits: s.asumptions, weights: weights})
+	sort.Sort(wLits{lits: s.hypothesis, weights: weights})
 	s.lastModel = make(Model, len(s.model))
 	var cost int
 	for status == Sat {
@@ -826,7 +900,7 @@ func (s *Solver) Minimize() int {
 		// Add a constraint incrementing current best cost
 		lits2 := make([]Lit, len(s.minLits))
 		weights2 := make([]int, len(s.minWeights))
-		copy(lits2, s.asumptions)
+		copy(lits2, s.hypothesis)
 		copy(weights2, weights)
 		s.AppendClause(NewPBClause(lits2, weights2, maxCost-cost+1))
 		s.rebuildOrderHeap()
@@ -835,7 +909,7 @@ func (s *Solver) Minimize() int {
 	return cost
 }
 
-// functions to sort asumptions for pseudo-boolean minimization clause.
+// functions to sort hypothesis for pseudo-boolean minimization clause.
 type wLits struct {
 	lits    []Lit
 	weights []int

vendor/github.com/crillab/gophersat/solver/types.go

@@ -42,6 +42,17 @@ type Var int32
 // Thus the CNF literal -3 is encoded as 2 * (3-1) + 1 = 5.
 type Lit int32
 
+// IntsToLits converts a list of CNF literals to a []Lit.
+// This is a helper function as the same result can be achieved by calling
+// IntToLit several times.
+func IntsToLits(vals ...int32) []Lit {
+	res := make([]Lit, len(vals))
+	for i, val := range vals {
+		res[i] = IntToLit(val)
+	}
+	return res
+}
+
 // IntToLit converts a CNF literal to a Lit.
 func IntToLit(i int32) Lit {
 	if i < 0 {

vendor/github.com/crillab/gophersat/solver/watcher.go

@@ -1,6 +1,9 @@
 package solver
 
-import "sort"
+import (
+	"fmt"
+	"sort"
+)
 
 type watcher struct {
 	other  Lit // Another lit from the clause
@@ -153,6 +156,26 @@ func (s *Solver) addLearned(c *Clause) {
 	s.wl.learned = append(s.wl.learned, c)
 	s.watchClause(c)
 	s.clauseBumpActivity(c)
+	if s.Certified {
+		if s.CertChan == nil {
+			fmt.Printf("%s\n", c.CNF())
+		} else {
+			s.CertChan <- c.CNF()
+		}
+	}
+}
+
+// Adds the given unit literal to the model at the top level.
+func (s *Solver) addLearnedUnit(unit Lit) {
+
+	s.model[unit.Var()] = lvlToSignedLvl(unit, 1)
+	if s.Certified {
+		if s.CertChan == nil {
+			fmt.Printf("%d 0\n", unit.Int())
+		} else {
+			s.CertChan <- fmt.Sprintf("%d 0", unit.Int())
+		}
+	}
 }
 
 // If l is negative, -lvl is returned. Else, lvl is returned.

vendor/github.com/mudler/topsort/topsort.go

@@ -16,6 +16,7 @@ package topsort
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 )
 
@@ -98,6 +99,7 @@ func (n node) edges() []string {
 	for k := range n {
 		keys = append(keys, k)
 	}
+	sort.Strings(keys)
 	return keys
 }
 

vendor/modules.txt

@@ -59,7 +59,7 @@ github.com/containerd/continuity/syscallx
 github.com/containerd/continuity/sysx
 # github.com/cpuguy83/go-md2man/v2 v2.0.0
 github.com/cpuguy83/go-md2man/v2/md2man
-# github.com/crillab/gophersat v1.1.9-0.20200211102949-9a8bf7f2f0a3
+# github.com/crillab/gophersat v1.3.2-0.20201023142334-3fc2ac466765
 github.com/crillab/gophersat/bf
 github.com/crillab/gophersat/solver
 # github.com/cyphar/filepath-securejoin v0.2.2
@@ -211,6 +211,8 @@ github.com/morikuni/aec
 github.com/mudler/cobra-extensions
 # github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
 github.com/mudler/docker-companion/api
+# github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+github.com/mudler/topsort
 # github.com/nxadm/tail v1.4.4
 github.com/nxadm/tail
 github.com/nxadm/tail/ratelimiter
@@ -303,8 +305,6 @@ github.com/spf13/jwalterweatherman
 github.com/spf13/pflag
 # github.com/spf13/viper v1.6.3
 github.com/spf13/viper
-# github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-github.com/stevenle/topsort
 # github.com/subosito/gotenv v1.2.0
 github.com/subosito/gotenv
 # github.com/urfave/cli v1.22.1